Skip to content

Commit

Permalink
[fix](hudi) fix fe memory leak (apache#41256)
Browse files Browse the repository at this point in the history
## Proposed changes
related pr: apache#41225
Should not set disable.cache to true, this will result in the
accumulation of com.aliyun.oss.ClientConfiguration.
<!--Describe your changes.-->
  • Loading branch information
suxiaogang223 authored Sep 27, 2024
1 parent 2bcec14 commit ab1061e
Showing 1 changed file with 0 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.time.LocalDateTime;
import java.time.ZoneId;
Expand Down Expand Up @@ -829,15 +828,6 @@ public static <T> T ugiDoAs(Configuration conf, PrivilegedExceptionAction<T> act
public static HoodieTableMetaClient getHudiClient(HMSExternalTable table) {
String hudiBasePath = table.getRemoteTable().getSd().getLocation();
Configuration conf = getConfiguration(table);
if (LOG.isDebugEnabled()) {
LOG.debug("try setting 'fs.xxx.impl.disable.cache' to true for hudi's base path: {}", hudiBasePath);
}
URI hudiBasePathUri = URI.create(hudiBasePath);
String scheme = hudiBasePathUri.getScheme();
if (!Strings.isNullOrEmpty(scheme)) {
// Avoid using Cache in Hadoop FileSystem, which may cause FE OOM.
conf.set("fs." + scheme + ".impl.disable.cache", "true");
}
return HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf),
() -> HoodieTableMetaClient.builder().setConf(conf).setBasePath(hudiBasePath).build());
}
Expand Down

0 comments on commit ab1061e

Please sign in to comment.