/vsis3/: advertize AWS_S3_ENDPOINT, and allow use of http:// / https:…

…// prefixing in it Fixes qgis/QGIS#60174 Funded by QGIS bugfixing program
rouault · Jan 21, 2025 · cca1ba9 · cca1ba9
1 parent 5212909
commit cca1ba9
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 6 deletions.
diff --git a/autotest/gcore/vsis3.py b/autotest/gcore/vsis3.py
@@ -128,9 +128,8 @@ def test_vsis3_init(aws_test_config):
 def test_vsis3_no_sign_request(aws_test_config_as_config_options_or_credentials):
 
     options = {
-        "AWS_S3_ENDPOINT": "s3.amazonaws.com",
+        "AWS_S3_ENDPOINT": "https://s3.amazonaws.com",
         "AWS_NO_SIGN_REQUEST": "YES",
-        "AWS_HTTPS": "YES",
         "AWS_VIRTUAL_HOSTING": "TRUE",
     }
 
@@ -175,7 +174,7 @@ def cbk(pct, _, tab):
 
     tab = [-1]
     options = {
-        "AWS_S3_ENDPOINT": "s3.amazonaws.com",
+        "AWS_S3_ENDPOINT": "http://s3.amazonaws.com",
         "AWS_NO_SIGN_REQUEST": "YES",
         "AWS_VIRTUAL_HOSTING": "FALSE",
     }

diff --git a/doc/source/user/virtual_file_systems.rst b/doc/source/user/virtual_file_systems.rst
@@ -568,14 +568,17 @@ The following configuration options are specific to the /vsis3/ handler:
       :default: s3.amazonaws.com
 
       Allows the use of /vsis3/ with non-AWS remote object stores that use the
-      AWS S3 protocol.
+      AWS S3 protocol. Starting with GDAL 3.11, this can be a URL starting
+      with ``http://`` or ``https://``.
 
 -  .. config:: AWS_HTTPS
       :choices: YES, NO
       :default: YES
 
       If ``YES``, AWS resources will be accessed using HTTPS. If ``NO``, HTTP
       will be used.
+      No longer needed starting with GDAL 3.11, because :config:`AWS_S3_ENDPOINT`
+      can include the protocol, and when doing so, this option is ignored.
 
 -  .. config:: AWS_VIRTUAL_HOSTING
       :choices: TRUE, FALSE

diff --git a/port/cpl_aws.cpp b/port/cpl_aws.cpp
@@ -2069,6 +2069,21 @@ VSIS3HandleHelper *VSIS3HandleHelper::BuildFromURI(const char *pszURI,
 
     std::string osEndpoint = VSIGetPathSpecificOption(
         osPathForOption.c_str(), "AWS_S3_ENDPOINT", "s3.amazonaws.com");
+    bool bForceHTTP = false;
+    bool bForceHTTPS = false;
+    if (STARTS_WITH(osEndpoint.c_str(), "http://"))
+    {
+        bForceHTTP = true;
+        osEndpoint = osEndpoint.substr(strlen("http://"));
+    }
+    else if (STARTS_WITH(osEndpoint.c_str(), "https://"))
+    {
+        bForceHTTPS = true;
+        osEndpoint = osEndpoint.substr(strlen("https://"));
+    }
+    if (!osEndpoint.empty() && osEndpoint.back() == '/')
+        osEndpoint.pop_back();
+
     if (!osRegion.empty() && osEndpoint == "s3.amazonaws.com")
     {
         osEndpoint = "s3." + osRegion + ".amazonaws.com";
@@ -2083,8 +2098,10 @@ VSIS3HandleHelper *VSIS3HandleHelper::BuildFromURI(const char *pszURI,
     {
         return nullptr;
     }
-    const bool bUseHTTPS = CPLTestBool(
-        VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_HTTPS", "YES"));
+    const bool bUseHTTPS =
+        bForceHTTPS ||
+        (!bForceHTTP && CPLTestBool(VSIGetPathSpecificOption(
+                            osPathForOption.c_str(), "AWS_HTTPS", "YES")));
     const bool bIsValidNameForVirtualHosting =
         osBucket.find('.') == std::string::npos;
     const bool bUseVirtualHosting = CPLTestBool(CSLFetchNameValueDef(

diff --git a/port/cpl_vsil_s3.cpp b/port/cpl_vsil_s3.cpp
@@ -1912,6 +1912,9 @@ const char *VSIS3FSHandler::GetOptions()
                 "  <Option name='AWS_REQUEST_PAYER' type='string' "
                 "description='Content of the x-amz-request-payer HTTP header. "
                 "Typically \"requester\" for requester-pays buckets'/>"
+                "  <Option name='AWS_S3_ENDPOINT' type='string' "
+                "description='Endpoint for a S3-compatible API' "
+                "default='https://s3.amazonaws.com'/>"
                 "  <Option name='AWS_VIRTUAL_HOSTING' type='boolean' "
                 "description='Whether to use virtual hosting server name when "
                 "the "