Skip to content

Commit

Permalink
Update gcs-connector to 3.0.4
Browse files Browse the repository at this point in the history
  • Loading branch information
@wendigo
wendigo committed Jan 30, 2025
1 parent f5790de commit 0ce646a
Show file tree
Hide file tree
Showing 4 changed files with 20 additions and 18 deletions.
4 changes: 3 additions & 1 deletion lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsAccessTokenProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
import com.google.cloud.hadoop.util.AccessTokenProvider;
import org.apache.hadoop.conf.Configuration;

import java.time.Instant;

import static com.google.common.base.Strings.nullToEmpty;
import static java.util.concurrent.TimeUnit.HOURS;

Expand All @@ -29,7 +31,7 @@ public class GcsAccessTokenProvider
@Override
public AccessToken getAccessToken()
{
return new AccessToken(nullToEmpty(config.get(GCS_ACCESS_TOKEN_CONF)), EXPIRATION_TIME_MILLISECONDS);
return new AccessToken(nullToEmpty(config.get(GCS_ACCESS_TOKEN_CONF)), Instant.now().plusMillis(EXPIRATION_TIME_MILLISECONDS));
}

@Override
Expand Down
18 changes: 8 additions & 10 deletions lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GcsStorageFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@
*/
package io.trino.hdfs.gcs;

import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.http.HttpTransport;
import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.json.jackson2.JacksonFactory;
import com.google.cloud.hadoop.repackaged.gcs.com.google.api.services.storage.Storage;
import com.google.cloud.hadoop.repackaged.gcs.com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.gcsio.GoogleCloudStorageOptions;
import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HttpTransportFactory;
import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.RetryHttpInitializer;
Expand All @@ -31,11 +31,10 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.time.Duration;
import java.util.Optional;

import static com.google.cloud.hadoop.fs.gcs.TrinoGoogleHadoopFileSystemConfiguration.getGcsOptionsBuilder;
import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.CredentialFactory.DEFAULT_SCOPES;
import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.CLOUD_PLATFORM_SCOPE;
import static com.google.common.base.Strings.nullToEmpty;
import static io.trino.hdfs.gcs.GcsConfigurationProvider.GCS_OAUTH_KEY;
import static java.nio.charset.StandardCharsets.UTF_8;
Expand All @@ -45,7 +44,7 @@ public class GcsStorageFactory
private static final String APPLICATION_NAME = "Trino";

private final boolean useGcsAccessToken;
private final Optional<GoogleCredential> jsonGoogleCredential;
private final Optional<GoogleCredentials> jsonGoogleCredential;

@Inject
public GcsStorageFactory(HiveGcsConfig hiveGcsConfig)
Expand All @@ -57,12 +56,12 @@ public GcsStorageFactory(HiveGcsConfig hiveGcsConfig)
String jsonKeyFilePath = hiveGcsConfig.getJsonKeyFilePath();
if (jsonKey != null) {
try (InputStream inputStream = new ByteArrayInputStream(jsonKey.getBytes(UTF_8))) {
jsonGoogleCredential = Optional.of(GoogleCredential.fromStream(inputStream).createScoped(DEFAULT_SCOPES));
jsonGoogleCredential = Optional.of(GoogleCredentials.fromStream(inputStream).createScoped(CLOUD_PLATFORM_SCOPE));
}
}
else if (jsonKeyFilePath != null) {
try (FileInputStream inputStream = new FileInputStream(jsonKeyFilePath)) {
jsonGoogleCredential = Optional.of(GoogleCredential.fromStream(inputStream).createScoped(DEFAULT_SCOPES));
jsonGoogleCredential = Optional.of(GoogleCredentials.fromStream(inputStream).createScoped(CLOUD_PLATFORM_SCOPE));
}
}
else {
Expand All @@ -75,22 +74,21 @@ public Storage create(HdfsEnvironment environment, HdfsContext context, Path pat
try {
GoogleCloudStorageOptions gcsOptions = getGcsOptionsBuilder(environment.getConfiguration(context, path)).build();
HttpTransport httpTransport = HttpTransportFactory.createHttpTransport(
gcsOptions.getTransportType(),
gcsOptions.getProxyAddress(),
gcsOptions.getProxyUsername(),
gcsOptions.getProxyPassword());
GoogleCredential credential;
GoogleCredentials credential;
if (useGcsAccessToken) {
String accessToken = nullToEmpty(context.getIdentity().getExtraCredentials().get(GCS_OAUTH_KEY));
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(accessToken.getBytes(UTF_8))) {
credential = GoogleCredential.fromStream(inputStream).createScoped(DEFAULT_SCOPES);
credential = GoogleCredentials.fromStream(inputStream).createScoped(CLOUD_PLATFORM_SCOPE);
}
}
else {
credential = jsonGoogleCredential.orElseThrow(() -> new IllegalStateException("GCS credentials not configured"));
}
return new Storage.Builder(httpTransport, JacksonFactory.getDefaultInstance(), new RetryHttpInitializer(credential, RetryHttpInitializerOptions.builder()
.setReadTimeout(Duration.ofMillis(gcsOptions.getHttpRequestReadTimeout()))
.setReadTimeout(gcsOptions.getHttpRequestReadTimeout())
.setMaxRequestRetries(gcsOptions.getMaxHttpRequestRetries())
.build()))
.setApplicationName(APPLICATION_NAME)
Expand Down
14 changes: 8 additions & 6 deletions lib/trino-hdfs/src/main/java/io/trino/hdfs/gcs/GoogleGcsConfigurationInitializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,11 @@
import java.util.Optional;

import static com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemConfiguration.GCS_CONFIG_PREFIX;
import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialConfiguration.ACCESS_TOKEN_PROVIDER_IMPL_SUFFIX;
import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialConfiguration.ENABLE_SERVICE_ACCOUNTS_SUFFIX;
import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialConfiguration.SERVICE_ACCOUNT_JSON_KEYFILE_SUFFIX;
import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.ACCESS_TOKEN_PROVIDER_SUFFIX;
import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AUTHENTICATION_TYPE_SUFFIX;
import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AuthenticationType.ACCESS_TOKEN_PROVIDER;
import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AuthenticationType.SERVICE_ACCOUNT_JSON_KEYFILE;
import static com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.SERVICE_ACCOUNT_JSON_KEYFILE_SUFFIX;
import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;

Expand Down Expand Up @@ -72,12 +74,12 @@ public void initializeConfiguration(Configuration config)

if (useGcsAccessToken) {
// use oauth token to authenticate with Google Cloud Storage
config.setBoolean(GCS_CONFIG_PREFIX + ENABLE_SERVICE_ACCOUNTS_SUFFIX.getKey(), false);
config.setClass(GCS_CONFIG_PREFIX + ACCESS_TOKEN_PROVIDER_IMPL_SUFFIX.getKey(), GcsAccessTokenProvider.class, AccessTokenProvider.class);
config.setEnum(GCS_CONFIG_PREFIX + AUTHENTICATION_TYPE_SUFFIX.getKey(), ACCESS_TOKEN_PROVIDER);
config.setClass(GCS_CONFIG_PREFIX + ACCESS_TOKEN_PROVIDER_SUFFIX.getKey(), GcsAccessTokenProvider.class, AccessTokenProvider.class);
}
else if (jsonKeyFilePath != null) {
// use service account key file
config.setBoolean(GCS_CONFIG_PREFIX + ENABLE_SERVICE_ACCOUNTS_SUFFIX.getKey(), true);
config.setEnum(GCS_CONFIG_PREFIX + AUTHENTICATION_TYPE_SUFFIX.getKey(), SERVICE_ACCOUNT_JSON_KEYFILE);
config.set(GCS_CONFIG_PREFIX + SERVICE_ACCOUNT_JSON_KEYFILE_SUFFIX.getKey(), jsonKeyFilePath);
}
}
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -472,7 +472,7 @@
<dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>gcs-connector</artifactId>
<version>hadoop3-2.2.26</version>
<version>3.0.4</version>
<classifier>shaded</classifier>
<exclusions>
<exclusion>
Expand Down

0 comments on commit 0ce646a

Please sign in to comment.