From 961dd0c903a9152c3e552cd96a31cda2ead7c3e2 Mon Sep 17 00:00:00 2001 From: Edward Ma <63481447+sfc-gh-ema@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:42:17 -0800 Subject: [PATCH] Fix cn aws url (#596) * Check for China region whenn determining the AWS URL * Fix domain URL * Update version number in preparation for release version tagging --- build.sbt | 2 +- .../spark/snowflake/io/CloudStorageOperations.scala | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index aaf2cedc..60147773 100644 --- a/build.sbt +++ b/build.sbt @@ -26,7 +26,7 @@ val testSparkVersion = sys.props.get("spark.testVersion").getOrElse("3.5.0") * Tests/jenkins/BumpUpSparkConnectorVersion/run.sh * in snowflake repository. */ -val sparkConnectorVersion = "3.1.0" +val sparkConnectorVersion = "3.1.1" lazy val ItTest = config("it") extend Test diff --git a/src/main/scala/net/snowflake/spark/snowflake/io/CloudStorageOperations.scala b/src/main/scala/net/snowflake/spark/snowflake/io/CloudStorageOperations.scala index 5f551c96..3c7a775f 100644 --- a/src/main/scala/net/snowflake/spark/snowflake/io/CloudStorageOperations.scala +++ b/src/main/scala/net/snowflake/spark/snowflake/io/CloudStorageOperations.scala @@ -406,9 +406,15 @@ object CloudStorageOperations { s3ClientBuilder.withEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration(stageEndPoint.get, regionName.get)) } else if (useRegionUrl.get.toBoolean) { + val awsDomain: String = + if (regionName.get.toLowerCase().startsWith("cn-")) { + ".amazonaws.com.cn" + } else { + ".amazonaws.com" + } s3ClientBuilder.withEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration( - "s3." + regionName.get + ".amazonaws.com", regionName.get)) + "s3." + regionName.get + awsDomain, regionName.get)) } else { s3ClientBuilder.withRegion(regionName.get) }