From 22fe2cab0b670fec5d56a94dcdcd97ea9ed2e7f6 Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 14 Jan 2025 16:55:28 +0800 Subject: [PATCH 1/4] add credential vending document for fileset --- docs/hadoop-catalog-with-adls.md | 22 +++++++++++++++++++++- docs/hadoop-catalog-with-gcs.md | 18 +++++++++++++++++- docs/hadoop-catalog-with-oss.md | 22 +++++++++++++++++++++- docs/hadoop-catalog-with-s3.md | 22 +++++++++++++++++++++- 4 files changed, 80 insertions(+), 4 deletions(-) diff --git a/docs/hadoop-catalog-with-adls.md b/docs/hadoop-catalog-with-adls.md index 96126c6fab9..5b894ffe6a9 100644 --- a/docs/hadoop-catalog-with-adls.md +++ b/docs/hadoop-catalog-with-adls.md @@ -482,7 +482,27 @@ Since 0.8.0-incubating, Gravitino supports credential vending for ADLS fileset. ### How to create an ADLS Hadoop catalog with credential enabled -Apart from configuration method in [create-adls-hadoop-catalog](#configuration-for-a-adls-hadoop-catalog), properties needed by [adls-credential](./security/credential-vending.md#adls-credentials) should also be set to enable credential vending for ADLS fileset. +Apart from configuration method in [create-adls-hadoop-catalog](#configuration-for-a-adls-hadoop-catalog), properties needed by [adls-credential](./security/credential-vending.md#adls-credentials) should also be set to enable credential vending for ADLS fileset. Take `adls-token` credential provider for example: + +```shell +curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "name": "example_catalog", + "type": "FILESET", + "comment": "This is a ADLS fileset catalog", + "provider": "hadoop", + "properties": { + "location": "abfss://container@account-name.dfs.core.windows.net/path", + "azure-storage-account-name": "The account name of the Azure Blob Storage", + "azure-storage-account-key": "The account key of the Azure Blob Storage", + "filesystem-providers": "abs", + "credential-providers": "adls-token", + "azure-tenant-id":"The Azure tenant id", + "azure-client-id":"The Azure client id", + "azure-client-secret":"The Azure client secret key" + } +}' http://localhost:8090/api/metalakes/metalake/catalogs +``` ### How to access ADLS fileset with credential diff --git a/docs/hadoop-catalog-with-gcs.md b/docs/hadoop-catalog-with-gcs.md index a3eb034b4fe..321c5392867 100644 --- a/docs/hadoop-catalog-with-gcs.md +++ b/docs/hadoop-catalog-with-gcs.md @@ -461,7 +461,23 @@ Since 0.8.0-incubating, Gravitino supports credential vending for GCS fileset. I ### How to create a GCS Hadoop catalog with credential enabled -Apart from configuration method in [create-gcs-hadoop-catalog](#configurations-for-a-gcs-hadoop-catalog), properties needed by [gcs-credential](./security/credential-vending.md#gcs-credentials) should also be set to enable credential vending for GCS fileset. +Apart from configuration method in [create-gcs-hadoop-catalog](#configurations-for-a-gcs-hadoop-catalog), properties needed by [gcs-credential](./security/credential-vending.md#gcs-credentials) should also be set to enable credential vending for GCS fileset. Take `gcs-token` credential provider for example: + +```shell +curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "name": "test_catalog", + "type": "FILESET", + "comment": "This is a GCS fileset catalog", + "provider": "hadoop", + "properties": { + "location": "gs://bucket/root", + "gcs-service-account-file": "path_of_gcs_service_account_file", + "filesystem-providers": "gcs", + "credential-providers": "gcs-token" + } +}' http://localhost:8090/api/metalakes/metalake/catalogs +``` ### How to access GCS fileset with credential diff --git a/docs/hadoop-catalog-with-oss.md b/docs/hadoop-catalog-with-oss.md index e63935c720a..6f7f39f3d77 100644 --- a/docs/hadoop-catalog-with-oss.md +++ b/docs/hadoop-catalog-with-oss.md @@ -497,7 +497,27 @@ Since 0.8.0-incubating, Gravitino supports credential vending for OSS fileset. I ### How to create a OSS Hadoop catalog with credential enabled -Apart from configuration method in [create-oss-hadoop-catalog](#configuration-for-an-oss-hadoop-catalog), properties needed by [oss-credential](./security/credential-vending.md#oss-credentials) should also be set to enable credential vending for OSS fileset. +Apart from configuration method in [create-oss-hadoop-catalog](#configuration-for-an-oss-hadoop-catalog), properties needed by [oss-credential](./security/credential-vending.md#oss-credentials) should also be set to enable credential vending for OSS fileset. Take `oss-token` credential provider for example: + +```shell +curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "name": "test_catalog", + "type": "FILESET", + "comment": "This is a OSS fileset catalog", + "provider": "hadoop", + "properties": { + "location": "oss://bucket/root", + "oss-access-key-id": "access_key", + "oss-secret-access-key": "secret_key", + "oss-endpoint": "http://oss-cn-hangzhou.aliyuncs.com", + "filesystem-providers": "oss", + "credential-providers": "oss-token", + "oss-region":"oss-cn-hangzhou", + "oss-role-arn":"The ARN of the role to access the OSS data" + } +}' http://localhost:8090/api/metalakes/metalake/catalogs +``` ### How to access OSS fileset with credential diff --git a/docs/hadoop-catalog-with-s3.md b/docs/hadoop-catalog-with-s3.md index 7d56f2b9ab8..8c9ea326495 100644 --- a/docs/hadoop-catalog-with-s3.md +++ b/docs/hadoop-catalog-with-s3.md @@ -500,7 +500,27 @@ Since 0.8.0-incubating, Gravitino supports credential vending for S3 fileset. If ### How to create a S3 Hadoop catalog with credential enabled -Apart from configuration method in [create-s3-hadoop-catalog](#configurations-for-s3-hadoop-catalog), properties needed by [s3-credential](./security/credential-vending.md#s3-credentials) should also be set to enable credential vending for S3 fileset. +Apart from configuration method in [create-s3-hadoop-catalog](#configurations-for-s3-hadoop-catalog), properties needed by [s3-credential](./security/credential-vending.md#s3-credentials) should also be set to enable credential vending for S3 fileset. Take `s3-token` credential provider for example: + +```shell +curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "name": "test_catalog", + "type": "FILESET", + "comment": "This is a S3 fileset catalog", + "provider": "hadoop", + "properties": { + "location": "s3a://bucket/root", + "s3-access-key-id": "access_key", + "s3-secret-access-key": "secret_key", + "s3-endpoint": "http://s3.ap-northeast-1.amazonaws.com", + "filesystem-providers": "s3", + "credential-providers": "s3-token", + "s3-region":"ap-northeast-1", + "s3-role-arn":"The ARN of the role to access the S3 data", + } +}' http://localhost:8090/api/metalakes/metalake/catalogs +``` ### How to access S3 fileset with credential From 5e499db6ae3cc802d692d720d78f0bafb16753d4 Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 14 Jan 2025 18:55:11 +0800 Subject: [PATCH 2/4] add credential vending document for fileset --- docs/hadoop-catalog-with-adls.md | 4 ++-- docs/hadoop-catalog-with-gcs.md | 4 ++-- docs/hadoop-catalog-with-oss.md | 4 ++-- docs/hadoop-catalog-with-s3.md | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/hadoop-catalog-with-adls.md b/docs/hadoop-catalog-with-adls.md index 5b894ffe6a9..1c958bd0cb3 100644 --- a/docs/hadoop-catalog-with-adls.md +++ b/docs/hadoop-catalog-with-adls.md @@ -480,7 +480,7 @@ For other use cases, please refer to the [Gravitino Virtual File System](./how-t Since 0.8.0-incubating, Gravitino supports credential vending for ADLS fileset. If the catalog has been [configured with credential](./security/credential-vending.md), you can access ADLS fileset without providing authentication information like `azure-storage-account-name` and `azure-storage-account-key` in the properties. -### How to create an ADLS Hadoop catalog with credential enabled +### How to create an ADLS Hadoop catalog with credential vending Apart from configuration method in [create-adls-hadoop-catalog](#configuration-for-a-adls-hadoop-catalog), properties needed by [adls-credential](./security/credential-vending.md#adls-credentials) should also be set to enable credential vending for ADLS fileset. Take `adls-token` credential provider for example: @@ -504,7 +504,7 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ }' http://localhost:8090/api/metalakes/metalake/catalogs ``` -### How to access ADLS fileset with credential +### How to access ADLS fileset with credential vending If the catalog has been configured with credential, you can access ADLS fileset without providing authentication information via GVFS Java/Python client and Spark. Let's see how to access ADLS fileset with credential: diff --git a/docs/hadoop-catalog-with-gcs.md b/docs/hadoop-catalog-with-gcs.md index 321c5392867..8db43a335da 100644 --- a/docs/hadoop-catalog-with-gcs.md +++ b/docs/hadoop-catalog-with-gcs.md @@ -459,7 +459,7 @@ For other use cases, please refer to the [Gravitino Virtual File System](./how-t Since 0.8.0-incubating, Gravitino supports credential vending for GCS fileset. If the catalog has been [configured with credential](./security/credential-vending.md), you can access GCS fileset without providing authentication information like `gcs-service-account-file` in the properties. -### How to create a GCS Hadoop catalog with credential enabled +### How to create a GCS Hadoop catalog with credential vending Apart from configuration method in [create-gcs-hadoop-catalog](#configurations-for-a-gcs-hadoop-catalog), properties needed by [gcs-credential](./security/credential-vending.md#gcs-credentials) should also be set to enable credential vending for GCS fileset. Take `gcs-token` credential provider for example: @@ -479,7 +479,7 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ }' http://localhost:8090/api/metalakes/metalake/catalogs ``` -### How to access GCS fileset with credential +### How to access GCS fileset with credential vending If the catalog has been configured with credential, you can access GCS fileset without providing authentication information via GVFS Java/Python client and Spark. Let's see how to access GCS fileset with credential: diff --git a/docs/hadoop-catalog-with-oss.md b/docs/hadoop-catalog-with-oss.md index 6f7f39f3d77..178d412012f 100644 --- a/docs/hadoop-catalog-with-oss.md +++ b/docs/hadoop-catalog-with-oss.md @@ -495,7 +495,7 @@ For other use cases, please refer to the [Gravitino Virtual File System](./how-t Since 0.8.0-incubating, Gravitino supports credential vending for OSS fileset. If the catalog has been [configured with credential](./security/credential-vending.md), you can access OSS fileset without providing authentication information like `oss-access-key-id` and `oss-secret-access-key` in the properties. -### How to create a OSS Hadoop catalog with credential enabled +### How to create an OSS Hadoop catalog with credential vending Apart from configuration method in [create-oss-hadoop-catalog](#configuration-for-an-oss-hadoop-catalog), properties needed by [oss-credential](./security/credential-vending.md#oss-credentials) should also be set to enable credential vending for OSS fileset. Take `oss-token` credential provider for example: @@ -519,7 +519,7 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ }' http://localhost:8090/api/metalakes/metalake/catalogs ``` -### How to access OSS fileset with credential +### How to access OSS fileset with credential vending If the catalog has been configured with credential, you can access OSS fileset without providing authentication information via GVFS Java/Python client and Spark. Let's see how to access OSS fileset with credential: diff --git a/docs/hadoop-catalog-with-s3.md b/docs/hadoop-catalog-with-s3.md index 8c9ea326495..179aabde18b 100644 --- a/docs/hadoop-catalog-with-s3.md +++ b/docs/hadoop-catalog-with-s3.md @@ -498,7 +498,7 @@ For more use cases, please refer to the [Gravitino Virtual File System](./how-to Since 0.8.0-incubating, Gravitino supports credential vending for S3 fileset. If the catalog has been [configured with credential](./security/credential-vending.md), you can access S3 fileset without providing authentication information like `s3-access-key-id` and `s3-secret-access-key` in the properties. -### How to create a S3 Hadoop catalog with credential enabled +### How to create a S3 Hadoop catalog with credential vending Apart from configuration method in [create-s3-hadoop-catalog](#configurations-for-s3-hadoop-catalog), properties needed by [s3-credential](./security/credential-vending.md#s3-credentials) should also be set to enable credential vending for S3 fileset. Take `s3-token` credential provider for example: @@ -522,7 +522,7 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ }' http://localhost:8090/api/metalakes/metalake/catalogs ``` -### How to access S3 fileset with credential +### How to access S3 fileset with credential vending If the catalog has been configured with credential, you can access S3 fileset without providing authentication information via GVFS Java/Python client and Spark. Let's see how to access S3 fileset with credential: From 037e4af3f13440d7eb4fef3a5241fd4b28489021 Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 14 Jan 2025 18:58:09 +0800 Subject: [PATCH 3/4] add credential vending document for fileset --- docs/hadoop-catalog-with-s3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/hadoop-catalog-with-s3.md b/docs/hadoop-catalog-with-s3.md index 179aabde18b..7df25d1605b 100644 --- a/docs/hadoop-catalog-with-s3.md +++ b/docs/hadoop-catalog-with-s3.md @@ -517,7 +517,7 @@ curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ "filesystem-providers": "s3", "credential-providers": "s3-token", "s3-region":"ap-northeast-1", - "s3-role-arn":"The ARN of the role to access the S3 data", + "s3-role-arn":"The ARN of the role to access the S3 data" } }' http://localhost:8090/api/metalakes/metalake/catalogs ``` From 46291d802c63b0115af7d650d7c861ef127f2c4e Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 14 Jan 2025 20:51:01 +0800 Subject: [PATCH 4/4] add credential vending document for fileset --- docs/hadoop-catalog-with-adls.md | 2 +- docs/hadoop-catalog-with-gcs.md | 2 +- docs/hadoop-catalog-with-oss.md | 2 +- docs/hadoop-catalog-with-s3.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/hadoop-catalog-with-adls.md b/docs/hadoop-catalog-with-adls.md index 1c958bd0cb3..880166776fd 100644 --- a/docs/hadoop-catalog-with-adls.md +++ b/docs/hadoop-catalog-with-adls.md @@ -487,7 +487,7 @@ Apart from configuration method in [create-adls-hadoop-catalog](#configuration-f ```shell curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ -H "Content-Type: application/json" -d '{ - "name": "example_catalog", + "name": "adls-catalog-with-token", "type": "FILESET", "comment": "This is a ADLS fileset catalog", "provider": "hadoop", diff --git a/docs/hadoop-catalog-with-gcs.md b/docs/hadoop-catalog-with-gcs.md index 8db43a335da..5422047efd8 100644 --- a/docs/hadoop-catalog-with-gcs.md +++ b/docs/hadoop-catalog-with-gcs.md @@ -466,7 +466,7 @@ Apart from configuration method in [create-gcs-hadoop-catalog](#configurations-f ```shell curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ -H "Content-Type: application/json" -d '{ - "name": "test_catalog", + "name": "gcs-catalog-with-token", "type": "FILESET", "comment": "This is a GCS fileset catalog", "provider": "hadoop", diff --git a/docs/hadoop-catalog-with-oss.md b/docs/hadoop-catalog-with-oss.md index 178d412012f..b9ef5f44e27 100644 --- a/docs/hadoop-catalog-with-oss.md +++ b/docs/hadoop-catalog-with-oss.md @@ -502,7 +502,7 @@ Apart from configuration method in [create-oss-hadoop-catalog](#configuration-fo ```shell curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ -H "Content-Type: application/json" -d '{ - "name": "test_catalog", + "name": "oss-catalog-with-token", "type": "FILESET", "comment": "This is a OSS fileset catalog", "provider": "hadoop", diff --git a/docs/hadoop-catalog-with-s3.md b/docs/hadoop-catalog-with-s3.md index 7df25d1605b..f1382761894 100644 --- a/docs/hadoop-catalog-with-s3.md +++ b/docs/hadoop-catalog-with-s3.md @@ -505,7 +505,7 @@ Apart from configuration method in [create-s3-hadoop-catalog](#configurations-fo ```shell curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ -H "Content-Type: application/json" -d '{ - "name": "test_catalog", + "name": "s3-catalog-with-token", "type": "FILESET", "comment": "This is a S3 fileset catalog", "provider": "hadoop",