From 614017afde35e8b9a5b0c73e7631e3a752ddfa67 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Wed, 23 Oct 2024 17:04:38 +0200 Subject: [PATCH 1/4] Clarify S3 usage --- json-schema/platforms/custom-s3.json | 7 +++++- platforms/aws-s3.md | 10 +++++--- platforms/custom-s3.md | 35 +++++++++++++++++++++++++++- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/json-schema/platforms/custom-s3.json b/json-schema/platforms/custom-s3.json index 25a89b7..632ae12 100644 --- a/json-schema/platforms/custom-s3.json +++ b/json-schema/platforms/custom-s3.json @@ -11,6 +11,11 @@ } }, "then": { - "$comment": "No specific validation rules apply" + "bucket": { + "type": "string" + }, + "region": { + "type": "string" + } } } \ No newline at end of file diff --git a/platforms/aws-s3.md b/platforms/aws-s3.md index 93fde52..1a32963 100644 --- a/platforms/aws-s3.md +++ b/platforms/aws-s3.md @@ -2,6 +2,10 @@ This defines the Amazon Web Services (AWS) S3 interface. -- `platform`: `https://{bucket}.s3.{region}.amazonaws.com` -- `bucket`: The bucket name -- `region`: One of the S3 regions (lowercase) +- `platform`: `https://{bucket}.s3.{region}.amazonaws.com`, + which is the endpoint URL after replacing all variables in the URL. +- `bucket`: The bucket name. +- `region`: One of the S3 regions (lowercase). + +**Note:** If `s3` exists in `auth:refs`, you should use sign requests, +e.g. using the AWS CLI parameter `--no-sign-request`. diff --git a/platforms/custom-s3.md b/platforms/custom-s3.md index c6c5ebd..e99db5d 100644 --- a/platforms/custom-s3.md +++ b/platforms/custom-s3.md @@ -2,4 +2,37 @@ This defines the S3 interface for providers other than AWS (e.g. minio-based). -- `platform`: The API URL, must be the endpoint URL that can be used for the AWS CLI for example. +- `platform`: The API URL (template), must be the endpoint URL that can be used for the AWS CLI for example, e.g. `https://{bucket}.example.com` or `http://example.com:9000`. +- `bucket`: The bucket name, if applicable. +- `region`: The region, if applicable. + +## Mapping to S3 tooling + +### GDAL (`/vsis3/`) + +GDAL documentation: + +- `platform`: Some options for S3 can be inferred from the given URL (template): + - `AWS_HTTPS` can be retrieved by parsing the scheme part of the URL. `https` = `ON`, `http` = `OFF`. + - `AWS_S3_ENDPOINT` is the authority part of the URL after replacing all variables in the URL. + - `AWS_VIRTUAL_HOSTING` must be set to `FALSE` if there's no `{bucket}` placeholder in the URL template, otherwise `TRUE` (default value). +- The `region` property corresponds to the `AWS_REGION` option. +- The `requester_pays` property corresponds to the `AWS_REQUEST_PAYER` option. If `requester_pays` is `true`, set `AWS_REQUEST_PAYER` to `requester`. +- If `s3` exists in `auth:refs`, you should set `AWS_NO_SIGN_REQUEST` to `YES`. Otherwise it should be `NO`. + +### AWS CLI + +AWS CLI documentation: + +- `platform` corresponds to `--endpoint-url` after replacing all variables in the URL. +- `region` corresponds to `--region`. +- If `s3` is **missing** from `auth:refs`, you should use `--no-sign-request`. + +### s3cmd + +s3cmd documentation: + +- `platform` corresponds to `--host` / `--host-bucket` after replacing all variables in the URL (?). +- `region` corresponds to `--region`. +- `requester_pays` corresponds to `--requester-pays`. +- If `s3` exists in `auth:refs`, you should use the `s3cmd sign` command. From 8ca06e459326cce26efa70793733e109ece65615 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Sun, 24 Nov 2024 19:57:02 +0100 Subject: [PATCH 2/4] Apply suggestions from code review --- platforms/aws-s3.md | 2 +- platforms/custom-s3.md | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/platforms/aws-s3.md b/platforms/aws-s3.md index 1a32963..08751fd 100644 --- a/platforms/aws-s3.md +++ b/platforms/aws-s3.md @@ -7,5 +7,5 @@ This defines the Amazon Web Services (AWS) S3 interface. - `bucket`: The bucket name. - `region`: One of the S3 regions (lowercase). -**Note:** If `s3` exists in `auth:refs`, you should use sign requests, +**Note:** If the `s3` authentication scheme (i.e. "Simple S3 authentication") is referred to through `auth:refs`, you should disable signing requests, e.g. using the AWS CLI parameter `--no-sign-request`. diff --git a/platforms/custom-s3.md b/platforms/custom-s3.md index e99db5d..5fcbec5 100644 --- a/platforms/custom-s3.md +++ b/platforms/custom-s3.md @@ -14,11 +14,12 @@ GDAL documentation: - `platform` corresponds to `--host` / `--host-bucket` after replacing all variables in the URL (?). - `region` corresponds to `--region`. - `requester_pays` corresponds to `--requester-pays`. -- If `s3` exists in `auth:refs`, you should use the `s3cmd sign` command. +- If the `s3` authentication scheme (i.e. "Simple S3 authentication") is referred to through `auth:refs`, + you should provide an secret access key and an access key id through environment variables, a profile or the `s3cmd sign` command. From 7656b81e095060d3e607452003bf06f20d0d7425 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Sun, 24 Nov 2024 20:02:16 +0100 Subject: [PATCH 3/4] Update platforms/custom-s3.md --- platforms/custom-s3.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platforms/custom-s3.md b/platforms/custom-s3.md index 5fcbec5..3b15426 100644 --- a/platforms/custom-s3.md +++ b/platforms/custom-s3.md @@ -14,7 +14,8 @@ GDAL documentation: Date: Mon, 9 Dec 2024 18:04:19 +0100 Subject: [PATCH 4/4] Clarify --- platforms/custom-s3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platforms/custom-s3.md b/platforms/custom-s3.md index 3b15426..459f356 100644 --- a/platforms/custom-s3.md +++ b/platforms/custom-s3.md @@ -34,7 +34,7 @@ AWS CLI documentation: -- `platform` corresponds to `--host` / `--host-bucket` after replacing all variables in the URL (?). +- `platform` corresponds to `--host` after replacing all variables in the URL. - `region` corresponds to `--region`. - `requester_pays` corresponds to `--requester-pays`. - If the `s3` authentication scheme (i.e. "Simple S3 authentication") is referred to through `auth:refs`,