Skip to content

Commit

Permalink
[OSPP]Support Kubernetes ConfigMap for Apollo java, golang client (#79)
Browse files Browse the repository at this point in the history
Solve the problem of Apollo client configuration information files being lost due to server downtime or Pod restart in the Kubernetes environment. By using Kubernetes ConfigMap as a new persistent storage solution, the reliability and fault tolerance of configuration information are improved.

discussion apolloconfig/apollo#5210
dyx1234 authored Oct 30, 2024
1 parent e43ddf5 commit 67eb866
Showing 15 changed files with 1,053 additions and 5 deletions.
3 changes: 3 additions & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
@@ -5,9 +5,12 @@ Release Notes.
Apollo Java 2.4.0

------------------

* [Fix the Cannot enhance @Configuration bean definition issue](https://github.com/apolloconfig/apollo-java/pull/82)
* [Feature openapi query namespace support not fill item](https://github.com/apolloconfig/apollo-java/pull/83)
* [Add more observability in apollo config client](https://github.com/apolloconfig/apollo-java/pull/74)
* [Feature Support Kubernetes ConfigMap cache for Apollo java, golang client](https://github.com/apolloconfig/apollo-java/pull/79)


------------------
All issues and pull requests are [here](https://github.com/apolloconfig/apollo-java/milestone/4?closed=1)
5 changes: 5 additions & 0 deletions apollo-client/pom.xml
Original file line number Diff line number Diff line change
@@ -70,6 +70,11 @@
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>io.kubernetes</groupId>
<artifactId>client-java</artifactId>
<optional>true</optional>
</dependency>
<!-- test -->
<dependency>
<groupId>org.eclipse.jetty</groupId>
Original file line number Diff line number Diff line change
@@ -22,7 +22,10 @@
* @since 1.1.0
*/
public enum ConfigSourceType {
REMOTE("Loaded from remote config service"), LOCAL("Loaded from local cache"), NONE("Load failed");
REMOTE("Loaded from remote config service"),
LOCAL("Loaded from local cache"),
CONFIGMAP("Loaded from k8s config map"),
NONE("Load failed");

private final String description;

Original file line number Diff line number Diff line change
@@ -0,0 +1,263 @@
/*
* Copyright 2022 Apollo Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.ctrip.framework.apollo.internals;

import com.ctrip.framework.apollo.kubernetes.KubernetesManager;
import com.ctrip.framework.apollo.build.ApolloInjector;
import com.ctrip.framework.apollo.core.ConfigConsts;
import com.ctrip.framework.apollo.core.utils.DeferredLoggerFactory;
import com.ctrip.framework.apollo.core.utils.StringUtils;
import com.ctrip.framework.apollo.enums.ConfigSourceType;
import com.ctrip.framework.apollo.exceptions.ApolloConfigException;
import com.ctrip.framework.apollo.tracer.Tracer;
import com.ctrip.framework.apollo.tracer.spi.Transaction;
import com.ctrip.framework.apollo.util.ConfigUtil;
import com.ctrip.framework.apollo.util.ExceptionUtil;
import com.ctrip.framework.apollo.util.escape.EscapeUtil;
import com.google.common.base.Preconditions;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.slf4j.Logger;

import java.lang.reflect.Type;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

/**
* @author dyx1234
*/
public class K8sConfigMapConfigRepository extends AbstractConfigRepository
implements RepositoryChangeListener {
private static final Logger logger = DeferredLoggerFactory.getLogger(K8sConfigMapConfigRepository.class);
private final String namespace;
private String configMapName;
private String configMapKey;
private final String k8sNamespace;
private final ConfigUtil configUtil;
private final KubernetesManager kubernetesManager;
private volatile Properties configMapProperties;
private volatile ConfigRepository upstream;
private volatile ConfigSourceType sourceType = ConfigSourceType.CONFIGMAP;
private static final Gson GSON = new Gson();


public K8sConfigMapConfigRepository(String namespace, ConfigRepository upstream) {
this.namespace = namespace;
configUtil = ApolloInjector.getInstance(ConfigUtil.class);
kubernetesManager = ApolloInjector.getInstance(KubernetesManager.class);
k8sNamespace = configUtil.getK8sNamespace();

this.setConfigMapKey(configUtil.getCluster(), namespace);
this.setConfigMapName(configUtil.getAppId(), false);
this.setUpstreamRepository(upstream);
}

private void setConfigMapKey(String cluster, String namespace) {
// cluster: User Definition >idc>default
if (StringUtils.isBlank(cluster)) {
configMapKey = EscapeUtil.createConfigMapKey("default", namespace);
return;
}
configMapKey = EscapeUtil.createConfigMapKey(cluster, namespace);
}

private void setConfigMapName(String appId, boolean syncImmediately) {
Preconditions.checkNotNull(appId, "AppId cannot be null");
configMapName = ConfigConsts.APOLLO_CONFIG_CACHE + appId;
this.checkConfigMapName(configMapName);
if (syncImmediately) {
this.sync();
}
}

private void checkConfigMapName(String configMapName) {
if (StringUtils.isBlank(configMapName)) {
throw new IllegalArgumentException("ConfigMap name cannot be null");
}
if (kubernetesManager.checkConfigMapExist(k8sNamespace, configMapName)) {
return;
}
// Create an empty configmap, write the new value in the update event
Transaction transaction = Tracer.newTransaction("Apollo.ConfigService", "createK8sConfigMap");
transaction.addData("configMapName", configMapName);
try {
kubernetesManager.createConfigMap(k8sNamespace, configMapName, null);
transaction.setStatus(Transaction.SUCCESS);
} catch (Throwable ex) {
Tracer.logEvent("ApolloConfigException", ExceptionUtil.getDetailMessage(ex));
transaction.setStatus(ex);
throw new ApolloConfigException("Create configmap failed!", ex);
} finally {
transaction.complete();
}
}

@Override
public Properties getConfig() {
if (configMapProperties == null) {
sync();
}
Properties result = propertiesFactory.getPropertiesInstance();
result.putAll(configMapProperties);
return result;
}

/**
* Update the memory when the configuration center changes
*
* @param upstreamConfigRepository the upstream repo
*/
@Override
public void setUpstreamRepository(ConfigRepository upstreamConfigRepository) {
if (upstreamConfigRepository == null) {
return;
}
//clear previous listener
if (upstream != null) {
upstream.removeChangeListener(this);
}
upstream = upstreamConfigRepository;
upstreamConfigRepository.addChangeListener(this);
}

@Override
public ConfigSourceType getSourceType() {
return sourceType;
}

/**
* Sync the configmap
*/
@Override
protected void sync() {
// Chain recovery, first read from upstream data source
boolean syncFromUpstreamResultSuccess = trySyncFromUpstream();

if (syncFromUpstreamResultSuccess) {
return;
}

Transaction transaction = Tracer.newTransaction("Apollo.ConfigService", "syncK8sConfigMap");
Throwable exception = null;
try {
configMapProperties = loadFromK8sConfigMap();
sourceType = ConfigSourceType.CONFIGMAP;
transaction.setStatus(Transaction.SUCCESS);
} catch (Throwable ex) {
Tracer.logEvent("ApolloConfigException", ExceptionUtil.getDetailMessage(ex));
transaction.setStatus(ex);
exception = ex;
} finally {
transaction.complete();
}

if (configMapProperties == null) {
sourceType = ConfigSourceType.NONE;
throw new ApolloConfigException(
"Load config from Kubernetes ConfigMap failed!", exception);
}
}

Properties loadFromK8sConfigMap() {
Preconditions.checkNotNull(configMapName, "ConfigMap name cannot be null");

try {
String jsonConfig = kubernetesManager.getValueFromConfigMap(k8sNamespace, configMapName, configMapKey);

// Convert jsonConfig to properties
Properties properties = propertiesFactory.getPropertiesInstance();
if (jsonConfig != null && !jsonConfig.isEmpty()) {
Type type = new TypeToken<Map<String, String>>() {}.getType();
Map<String, String> configMap = GSON.fromJson(jsonConfig, type);
configMap.forEach(properties::setProperty);
}
return properties;
} catch (Exception ex) {
Tracer.logError(ex);
throw new ApolloConfigException(String
.format("Load config from Kubernetes ConfigMap %s failed!", configMapName), ex);
}
}

private boolean trySyncFromUpstream() {
if (upstream == null) {
return false;
}
try {
updateConfigMapProperties(upstream.getConfig(), upstream.getSourceType());
return true;
} catch (Throwable ex) {
Tracer.logError(ex);
logger.warn("Sync config from upstream repository {} failed, reason: {}", upstream.getClass(),
ExceptionUtil.getDetailMessage(ex));
}
return false;
}

private synchronized void updateConfigMapProperties(Properties newProperties, ConfigSourceType sourceType) {
this.sourceType = sourceType;
if (newProperties == null || newProperties.equals(configMapProperties)) {
return;
}
this.configMapProperties = newProperties;
persistConfigMap(configMapProperties);
}

/**
* Update the memory
*
* @param namespace the namespace of this repository change
* @param newProperties the properties after change
*/
@Override
public void onRepositoryChange(String namespace, Properties newProperties) {
if (newProperties == null || newProperties.equals(configMapProperties)) {
return;
}
Properties newFileProperties = propertiesFactory.getPropertiesInstance();
newFileProperties.putAll(newProperties);
updateConfigMapProperties(newFileProperties, upstream.getSourceType());
this.fireRepositoryChange(namespace, newProperties);
}

void persistConfigMap(Properties properties) {
Transaction transaction = Tracer.newTransaction("Apollo.ConfigService", "persistK8sConfigMap");
transaction.addData("configMapName", configMapName);
transaction.addData("k8sNamespace", k8sNamespace);
try {
// Convert properties to a JSON string using Gson
String jsonConfig = GSON.toJson(properties);
Map<String, String> data = new HashMap<>();
data.put(configMapKey, jsonConfig);

// update configmap
kubernetesManager.updateConfigMap(k8sNamespace, configMapName, data);
transaction.setStatus(Transaction.SUCCESS);
} catch (Exception ex) {
ApolloConfigException exception =
new ApolloConfigException(
String.format("Persist config to Kubernetes ConfigMap %s failed!", configMapName), ex);
Tracer.logError(exception);
transaction.setStatus(exception);
logger.error("Persist config to Kubernetes ConfigMap failed!", exception);
} finally {
transaction.complete();
}
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
/*
* Copyright 2022 Apollo Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.ctrip.framework.apollo.kubernetes;

import com.ctrip.framework.apollo.core.utils.StringUtils;
import com.ctrip.framework.apollo.exceptions.ApolloConfigException;
import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.models.V1ConfigMap;
import io.kubernetes.client.openapi.models.V1ObjectMeta;
import io.kubernetes.client.util.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;

@Service
public class KubernetesManager {
private static final Logger logger = LoggerFactory.getLogger(KubernetesManager.class);

private ApiClient client;
private CoreV1Api coreV1Api;

public KubernetesManager() {
try {
client = Config.defaultClient();
coreV1Api = new CoreV1Api(client);
} catch (Exception e) {
String errorMessage = "Failed to initialize Kubernetes client: " + e.getMessage();
logger.error(errorMessage, e);
throw new RuntimeException(errorMessage, e);
}
}

public KubernetesManager(CoreV1Api coreV1Api) {
this.coreV1Api = coreV1Api;
}

private V1ConfigMap buildConfigMap(String name, String namespace, Map<String, String> data) {
V1ObjectMeta metadata = new V1ObjectMeta()
.name(name)
.namespace(namespace);

return new V1ConfigMap()
.apiVersion("v1")
.kind("ConfigMap")
.metadata(metadata)
.data(data);
}

/**
* Creates a Kubernetes ConfigMap.
*
* @param k8sNamespace the namespace of the ConfigMap
* @param name the name of the ConfigMap
* @param data the data to be stored in the ConfigMap
* @return the name of the created ConfigMap
* @throws RuntimeException if an error occurs while creating the ConfigMap
*/
public String createConfigMap(String k8sNamespace, String name, Map<String, String> data) {
if (StringUtils.isEmpty(k8sNamespace) || StringUtils.isEmpty(name)) {
logger.error("create configmap failed due to null or empty parameter: k8sNamespace={}, name={}", k8sNamespace, name);
return null;
}
V1ConfigMap configMap = buildConfigMap(name, k8sNamespace, data);
try {
coreV1Api.createNamespacedConfigMap(k8sNamespace, configMap, null, null, null, null);
logger.info("ConfigMap created successfully: name: {}, namespace: {}", name, k8sNamespace);
return name;
} catch (Exception e) {
logger.error("Failed to create ConfigMap: {}", e.getMessage(), e);
throw new RuntimeException("Failed to create ConfigMap: " + e.getMessage(), e);
}
}

/**
* get value from config map
*
* @param k8sNamespace k8sNamespace
* @param name config map name
* @param key config map key (cluster+namespace)
* @return value(json string)
*/
public String getValueFromConfigMap(String k8sNamespace, String name, String key) {
if (StringUtils.isEmpty(k8sNamespace) || StringUtils.isEmpty(name) || StringUtils.isEmpty(key)) {
logger.error("Parameters can not be null or empty: k8sNamespace={}, name={}", k8sNamespace, name);
return null;
}
try {
V1ConfigMap configMap = coreV1Api.readNamespacedConfigMap(name, k8sNamespace, null);
if (!Objects.requireNonNull(configMap.getData()).containsKey(key)) {
logger.error("Specified key not found in ConfigMap: {}, k8sNamespace: {}, name: {}", name, k8sNamespace, name);
}
return configMap.getData().get(key);
} catch (Exception e) {
logger.error("Error occurred while getting value from ConfigMap: {}", e.getMessage(), e);
return null;
}
}

/**
* update config map
*
* @param k8sNamespace configmap namespace
* @param name config map name
* @param data new data
* @return config map name
*/
// Set the retry times using the client retry mechanism (CAS)
public boolean updateConfigMap(String k8sNamespace, String name, Map<String, String> data) throws ApiException {
if (StringUtils.isEmpty(k8sNamespace) || StringUtils.isEmpty(name)) {
logger.error("Parameters can not be null or empty: k8sNamespace={}, name={}", k8sNamespace, name);
return false;
}

int maxRetries = 5;
int retryCount = 0;
long waitTime = 100;

while (retryCount < maxRetries) {
try {
V1ConfigMap configmap = coreV1Api.readNamespacedConfigMap(name, k8sNamespace, null);
Map<String, String> existingData = configmap.getData();
if (existingData == null) {
existingData = new HashMap<>();
}

// Determine if the data contains its own kv and de-weight it
Map<String, String> finalExistingData = existingData;
boolean containsEntry = data.entrySet().stream()
.allMatch(entry -> entry.getValue().equals(finalExistingData.get(entry.getKey())));

if (containsEntry) {
logger.info("Data is identical or already contains the entry, no update needed.");
return true;
}

// Add new entries to the existing data
existingData.putAll(data);
configmap.setData(existingData);

coreV1Api.replaceNamespacedConfigMap(name, k8sNamespace, configmap, null, null, null, null);
return true;
} catch (ApiException e) {
if (e.getCode() == 409) {
retryCount++;
logger.warn("Conflict occurred, retrying... ({})", retryCount);
try {
// Scramble the time, so that different machines in the distributed retry time is different
// The random ratio ranges from 0.9 to 1.1
TimeUnit.MILLISECONDS.sleep((long) (waitTime * (0.9 + Math.random() * 0.2)));
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
}
waitTime = Math.min(waitTime * 2, 1000);
} else {
logger.error("Error updating ConfigMap: {}", e.getMessage(), e);
throw e;
}
}
}
String errorMessage = String.format("Failed to update ConfigMap after %d retries: k8sNamespace=%s, name=%s", maxRetries, k8sNamespace, name);
logger.error(errorMessage);
throw new ApolloConfigException(errorMessage);
}

/**
* check config map exist
*
* @param k8sNamespace config map namespace
* @param configMapName config map name
* @return true if config map exist, false otherwise
*/
public boolean checkConfigMapExist(String k8sNamespace, String configMapName) {
if (StringUtils.isEmpty(k8sNamespace) || StringUtils.isEmpty(configMapName)) {
logger.error("Parameters can not be null or empty: k8sNamespace={}, configMapName={}", k8sNamespace, configMapName);
return false;
}
try {
logger.info("Check whether ConfigMap exists, configMapName: {}", configMapName);
coreV1Api.readNamespacedConfigMap(configMapName, k8sNamespace, null);
return true;
} catch (Exception e) {
// configmap not exist
logger.info("ConfigMap not existence");
return false;
}
}
}
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@
import com.ctrip.framework.apollo.internals.XmlConfigFile;
import com.ctrip.framework.apollo.internals.YamlConfigFile;
import com.ctrip.framework.apollo.internals.YmlConfigFile;
import com.ctrip.framework.apollo.internals.K8sConfigMapConfigRepository;
import com.ctrip.framework.apollo.util.ConfigUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -111,7 +112,9 @@ public ConfigFile createConfigFile(String namespace, ConfigFileFormat configFile
}

ConfigRepository createConfigRepository(String namespace) {
if (m_configUtil.isPropertyFileCacheEnabled()) {
if (m_configUtil.isPropertyKubernetesCacheEnabled()) {
return createConfigMapConfigRepository(namespace);
} else if (m_configUtil.isPropertyFileCacheEnabled()) {
return createLocalConfigRepository(namespace);
}
return createRemoteConfigRepository(namespace);
@@ -133,6 +136,15 @@ LocalFileConfigRepository createLocalConfigRepository(String namespace) {
return new LocalFileConfigRepository(namespace, createRemoteConfigRepository(namespace));
}

/**
* Creates a Kubernetes config map repository for a given namespace
* @param namespace the namespace of the repository
* @return the newly created repository for the given namespace
*/
private ConfigRepository createConfigMapConfigRepository(String namespace) {
return new K8sConfigMapConfigRepository(namespace, createLocalConfigRepository(namespace));
}

RemoteConfigRepository createRemoteConfigRepository(String namespace) {
return new RemoteConfigRepository(namespace);
}
Original file line number Diff line number Diff line change
@@ -71,6 +71,7 @@ public class ConfigUtil {
private boolean propertyNamesCacheEnabled = false;
private boolean propertyFileCacheEnabled = true;
private boolean overrideSystemProperties = true;
private boolean propertyKubernetesCacheEnabled = false;
private boolean clientMonitorEnabled = false;
private boolean clientMonitorJmxEnabled = false;
private String monitorExternalType = "NONE";
@@ -91,6 +92,7 @@ public ConfigUtil() {
initPropertyNamesCacheEnabled();
initPropertyFileCacheEnabled();
initOverrideSystemProperties();
initPropertyKubernetesCacheEnabled();
initClientMonitorEnabled();
initClientMonitorJmxEnabled();
initClientMonitorExternalType();
@@ -376,6 +378,34 @@ private String getDeprecatedCustomizedCacheRoot() {
return cacheRoot;
}

public String getK8sNamespace() {
String k8sNamespace = getCacheKubernetesNamespace();

if (!Strings.isNullOrEmpty(k8sNamespace)) {
return k8sNamespace;
}

return ConfigConsts.KUBERNETES_CACHE_CONFIG_MAP_NAMESPACE_DEFAULT;
}

private String getCacheKubernetesNamespace() {
// 1. Get from System Property
String k8sNamespace = System.getProperty(ApolloClientSystemConsts.APOLLO_CACHE_KUBERNETES_NAMESPACE);
if (Strings.isNullOrEmpty(k8sNamespace)) {
// 2. Get from OS environment variable
k8sNamespace = System.getenv(ApolloClientSystemConsts.APOLLO_CACHE_KUBERNETES_NAMESPACE_ENVIRONMENT_VARIABLES);
}
if (Strings.isNullOrEmpty(k8sNamespace)) {
// 3. Get from server.properties
k8sNamespace = Foundation.server().getProperty(ApolloClientSystemConsts.APOLLO_CACHE_KUBERNETES_NAMESPACE, null);
}
if (Strings.isNullOrEmpty(k8sNamespace)) {
// 4. Get from app.properties
k8sNamespace = Foundation.app().getProperty(ApolloClientSystemConsts.APOLLO_CACHE_KUBERNETES_NAMESPACE, null);
}
return k8sNamespace;
}

public boolean isInLocalMode() {
try {
return Env.LOCAL == getApolloEnv();
@@ -479,6 +509,10 @@ public boolean isPropertyFileCacheEnabled() {
return propertyFileCacheEnabled;
}

public boolean isPropertyKubernetesCacheEnabled() {
return propertyKubernetesCacheEnabled;
}

public boolean isOverrideSystemProperties() {
return overrideSystemProperties;
}
@@ -500,11 +534,15 @@ private void initOverrideSystemProperties() {
ApolloClientSystemConsts.APOLLO_OVERRIDE_SYSTEM_PROPERTIES,
overrideSystemProperties);
}



private void initPropertyKubernetesCacheEnabled() {
propertyKubernetesCacheEnabled = getPropertyBoolean(ApolloClientSystemConsts.APOLLO_KUBERNETES_CACHE_ENABLE,
ApolloClientSystemConsts.APOLLO_KUBERNETES_CACHE_ENABLE_ENVIRONMENT_VARIABLES,
propertyKubernetesCacheEnabled);
}

private void initClientMonitorExternalType() {
monitorExternalType = System.getProperty(ApolloClientSystemConsts.APOLLO_CLIENT_MONITOR_EXTERNAL_TYPE);

if (Strings.isNullOrEmpty(monitorExternalType)) {
monitorExternalType = Foundation.app()
.getProperty(ApolloClientSystemConsts.APOLLO_CLIENT_MONITOR_EXTERNAL_TYPE, "NONE");
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Copyright 2022 Apollo Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.ctrip.framework.apollo.util.escape;

/**
* @author dyx1234
*/
public class EscapeUtil {

private static final String SINGLE_UNDERSCORE = "_";
private static final String DOUBLE_UNDERSCORE = "__";
private static final String TRIPLE_UNDERSCORE = "___";

// Escapes a single underscore in a namespace
public static String escapeNamespace(String namespace) {
if (namespace == null || namespace.isEmpty()) {
throw new IllegalArgumentException("Namespace cannot be null or empty");
}
return namespace.replace(SINGLE_UNDERSCORE, DOUBLE_UNDERSCORE);
}

// Concatenate the cluster and the escaped namespace, using three underscores as delimiters
public static String createConfigMapKey(String cluster, String namespace) {
String escapedNamespace = escapeNamespace(namespace);
return String.join(TRIPLE_UNDERSCORE, cluster, escapedNamespace);
}

}
Original file line number Diff line number Diff line change
@@ -112,6 +112,20 @@
"description": "enable property names cache.",
"defaultValue": false
},
{
"name": "apollo.cache.kubernetes.enable",
"type": "java.lang.Boolean",
"sourceType": "com.ctrip.framework.apollo.util.ConfigUtil",
"description": "enable kubernetes configmap cache.",
"defaultValue": false
},
{
"name": "apollo.cache.kubernetes.namespace",
"type": "java.lang.String",
"sourceType": "com.ctrip.framework.apollo.util.ConfigUtil",
"description": "kubernetes configmap namespace.",
"defaultValue": "default"
},
{
"name": "apollo.property.order.enable",
"type": "java.lang.Boolean",
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
/*
* Copyright 2022 Apollo Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.ctrip.framework.apollo.internals;

import com.ctrip.framework.apollo.build.MockInjector;
import com.ctrip.framework.apollo.enums.ConfigSourceType;
import com.ctrip.framework.apollo.kubernetes.KubernetesManager;
import com.ctrip.framework.apollo.util.ConfigUtil;
import com.ctrip.framework.apollo.util.escape.EscapeUtil;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.models.V1ConfigMap;
import io.kubernetes.client.openapi.models.V1ObjectMeta;
import org.junit.Before;
import org.junit.Test;
import org.springframework.test.util.ReflectionTestUtils;

import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.*;

public class K8sConfigMapConfigRepositoryTest {
private static String someAppId = "someApp";
private static String someCluster = "someCluster";
private String someNamespace = "default";
private static final String someConfigmapName = "apollo-configcache-someApp";

private static final String defaultKey = "defaultKey";
private static final String defaultValue = "defaultValue";
private static final String defaultJsonValue = "{\"id\":123,\"name\":\"John Doe\",\"email\":\"john.doe@example.com\"}";

private ConfigRepository upstreamRepo;
private Properties someProperties;
private ConfigSourceType someSourceType = ConfigSourceType.LOCAL;
private V1ConfigMap configMap;
private Map<String, String> data;
private KubernetesManager kubernetesManager;
private K8sConfigMapConfigRepository k8sConfigMapConfigRepository;


@Before
public void setUp() {
// mock configUtil
MockInjector.setInstance(ConfigUtil.class, new MockConfigUtil());
// mock kubernetesManager
kubernetesManager = mock(KubernetesManager.class);
MockInjector.setInstance(KubernetesManager.class, kubernetesManager);

// mock upstream
someProperties = new Properties();
someProperties.setProperty(defaultKey, defaultValue);
upstreamRepo = mock(ConfigRepository.class);
when(upstreamRepo.getConfig()).thenReturn(someProperties);
when(upstreamRepo.getSourceType()).thenReturn(someSourceType);

// make configmap
data = new HashMap<>();
data.put(defaultKey, defaultJsonValue);
configMap = new V1ConfigMap()
.metadata(new V1ObjectMeta().name(someAppId).namespace(someNamespace))
.data(data);

k8sConfigMapConfigRepository = new K8sConfigMapConfigRepository(someNamespace, upstreamRepo);
}

/**
* 测试setConfigMapKey方法,当cluster和namespace都为正常值时
*/
@Test
public void testSetConfigMapKeyUnderNormalConditions() throws Throwable {
// arrange
String cluster = "testCluster";
String namespace = "test_Namespace_1";
String escapedKey = "testCluster___test__Namespace__1";

// act
ReflectionTestUtils.invokeMethod(k8sConfigMapConfigRepository, "setConfigMapKey", cluster, namespace);

// assert
String expectedConfigMapKey = EscapeUtil.createConfigMapKey(cluster, namespace);
assertEquals(escapedKey, ReflectionTestUtils.getField(k8sConfigMapConfigRepository, "configMapKey"));
assertEquals(expectedConfigMapKey, ReflectionTestUtils.getField(k8sConfigMapConfigRepository, "configMapKey"));
}

/**
* 测试sync方法成功从上游数据源同步
*/
@Test
public void testSyncSuccessFromUpstream() throws Throwable {
// arrange
k8sConfigMapConfigRepository.setUpstreamRepository(upstreamRepo);

// act
k8sConfigMapConfigRepository.sync();

// assert
verify(upstreamRepo, times(1)).getConfig();
}


/**
* 测试sync方法从上游数据源同步失败,成功从Kubernetes的ConfigMap中加载
*/
@Test
public void testSyncFailFromUpstreamSuccessFromConfigMap() throws Throwable {
// arrange
ConfigRepository upstream = mock(ConfigRepository.class);
when(upstream.getConfig()).thenThrow(new RuntimeException("Upstream sync failed"));
k8sConfigMapConfigRepository.setUpstreamRepository(upstream);
when(kubernetesManager.getValueFromConfigMap(anyString(), anyString(), anyString())).thenReturn(data.get(defaultKey));

// act
k8sConfigMapConfigRepository.sync();

// assert
verify(kubernetesManager, times(1)).getValueFromConfigMap(anyString(), anyString(), anyString());
}

@Test
public void testGetConfig() {
// Arrange
Properties expectedProperties = new Properties();
expectedProperties.setProperty(defaultKey, defaultValue);
when(upstreamRepo.getConfig()).thenReturn(expectedProperties);
// Act
Properties actualProperties = k8sConfigMapConfigRepository.getConfig();
// Assert
assertNotNull(actualProperties);
assertEquals(defaultValue, actualProperties.getProperty(defaultKey));
}

@Test
public void testPersistConfigMap() throws ApiException {
// Arrange
Properties properties = new Properties();
properties.setProperty(defaultKey, defaultValue);
// Act
k8sConfigMapConfigRepository.persistConfigMap(properties);
// Assert
verify(kubernetesManager, times(1)).updateConfigMap(anyString(), anyString(), anyMap());
}

@Test
public void testOnRepositoryChange() throws ApiException {
// Arrange
Properties newProperties = new Properties();
newProperties.setProperty(defaultKey, defaultValue);
// Act
k8sConfigMapConfigRepository.onRepositoryChange(someNamespace, newProperties);
// Assert
verify(kubernetesManager, times(1)).updateConfigMap(anyString(), anyString(), anyMap());
}

@Test
public void testLoadFromK8sConfigMapSuccess() {
when(kubernetesManager.getValueFromConfigMap(anyString(), anyString(), anyString())).thenReturn(defaultJsonValue);

Properties properties = k8sConfigMapConfigRepository.loadFromK8sConfigMap();

assertNotNull(properties);
}

public static class MockConfigUtil extends ConfigUtil {
@Override
public String getAppId() {
return someAppId;
}

@Override
public String getCluster() {
return someCluster;
}
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
/*
* Copyright 2022 Apollo Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.ctrip.framework.apollo.kubernetes;

import com.ctrip.framework.apollo.build.MockInjector;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.models.V1ConfigMap;
import io.kubernetes.client.openapi.models.V1ObjectMeta;
import org.junit.Before;
import org.junit.Test;

import java.util.HashMap;
import java.util.Map;

import static org.mockito.Mockito.*;
import static org.junit.Assert.*;

public class KubernetesManagerTest {

private CoreV1Api coreV1Api;
private KubernetesManager kubernetesManager;

@Before
public void setUp() {
coreV1Api = mock(CoreV1Api.class);
kubernetesManager = new KubernetesManager(coreV1Api);

MockInjector.setInstance(KubernetesManager.class, kubernetesManager);
MockInjector.setInstance(CoreV1Api.class, coreV1Api);
}

/**
* 测试 createConfigMap 成功创建配置
*/
@Test
public void testCreateConfigMapSuccess() throws Exception {
// arrange
String namespace = "default";
String name = "testConfigMap";
Map<String, String> data = new HashMap<>();
data.put("key", "value");
V1ConfigMap configMap = new V1ConfigMap()
.metadata(new V1ObjectMeta().name(name).namespace(namespace))
.data(data);

when(coreV1Api.createNamespacedConfigMap(eq(namespace), eq(configMap), isNull(), isNull(), isNull(),isNull())).thenReturn(configMap);

// act
String result = kubernetesManager.createConfigMap(namespace, name, data);

// assert
verify(coreV1Api, times(1)).createNamespacedConfigMap(eq(namespace), any(V1ConfigMap.class), isNull(), isNull(), isNull(),isNull());
assert name.equals(result);
}

/**
* 测试 createConfigMap 传入 null 作为数据,正常执行
*/
@Test
public void testCreateConfigMapNullData() throws Exception {
// arrange
String namespace = "default";
String name = "testConfigMap";
Map<String, String> data = null;

// act
String result = kubernetesManager.createConfigMap(namespace, name, data);

// assert
verify(coreV1Api, times(1)).createNamespacedConfigMap(eq(namespace), any(V1ConfigMap.class), isNull(), isNull(), isNull(),isNull());
assert name.equals(result);
}

/**
* 测试getValueFromConfigMap方法,当ConfigMap存在且包含指定key时返回正确的value
*/
@Test
public void testGetValueFromConfigMapReturnsValue() throws Exception {
// arrange
String namespace = "default";
String name = "testConfigMap";
String key = "testKey";
String expectedValue = "testValue";
V1ConfigMap configMap = new V1ConfigMap();
configMap.putDataItem(key, expectedValue);

when(coreV1Api.readNamespacedConfigMap(name, namespace, null)).thenReturn(configMap);

// act
String actualValue = kubernetesManager.getValueFromConfigMap(namespace, name, key);

// assert
assertEquals(expectedValue, actualValue);
}

/**
* 测试getValueFromConfigMap方法,当ConfigMap不存在指定key时返回null
*/
@Test
public void testGetValueFromConfigMapKeyNotFound() throws Exception {
// arrange
String namespace = "default";
String name = "testConfigMap";
String key = "nonExistingKey";
V1ConfigMap configMap = new V1ConfigMap();
when(coreV1Api.readNamespacedConfigMap(name, namespace, null)).thenReturn(configMap);

// act
String actualValue = kubernetesManager.getValueFromConfigMap(namespace, name, key);

// assert
assertNull(actualValue);
}

/**
* 测试updateConfigMap成功的情况
*/
@Test
public void testUpdateConfigMapSuccess() throws Exception {
// arrange
String namespace = "default";
String name = "testConfigMap";
Map<String, String> data = new HashMap<>();
data.put("key", "value");
V1ConfigMap configMap = new V1ConfigMap();
configMap.metadata(new V1ObjectMeta().name(name).namespace(namespace));
configMap.data(data);

when(coreV1Api.readNamespacedConfigMap(name, namespace, null)).thenReturn(configMap);
when(coreV1Api.replaceNamespacedConfigMap(name, namespace, configMap, null, null, null, null)).thenReturn(configMap);

// act
Boolean success = kubernetesManager.updateConfigMap(namespace, name, data);

// assert
assertTrue(success);
}

/**
* 测试ConfigMap存在时,checkConfigMapExist方法返回true
*/
@Test
public void testCheckConfigMapExistWhenConfigMapExists() throws Exception {
// arrange
String namespace = "default";
String name = "testConfigMap";

// 创建一个模拟的 V1ConfigMap 实例
V1ConfigMap mockConfigMap = new V1ConfigMap();
mockConfigMap.setMetadata(new V1ObjectMeta().name(name).namespace(namespace));
doReturn(mockConfigMap).when(coreV1Api).readNamespacedConfigMap(name, namespace, null);

// act
boolean result = kubernetesManager.checkConfigMapExist(namespace, name);

// assert
assertEquals(true, result);
}

/**
* 测试ConfigMap不存在的情况,返回false
*/
@Test
public void testCheckConfigMapExistWhenConfigMapDoesNotExist() throws Exception {
// arrange
String namespace = "default";
String name = "testConfigMap";
doThrow(new ApiException("ConfigMap not exist")).when(coreV1Api).readNamespacedConfigMap(name, namespace, null);

// act
boolean result = kubernetesManager.checkConfigMapExist(namespace, name);

// assert
assertFalse(result);
}

/**
* 测试参数k8sNamespace和configMapName都为空时,checkConfigMapExist方法返回false
*/
@Test
public void testCheckConfigMapExistWithEmptyNamespaceAndName() {
// arrange
String namespace = "";
String name = "";

// act
boolean result = kubernetesManager.checkConfigMapExist(namespace, name);

// assert
assertFalse(result);
}

}
Original file line number Diff line number Diff line change
@@ -47,6 +47,8 @@ public void tearDown() throws Exception {
System.clearProperty(ApolloClientSystemConsts.APOLLO_CACHE_DIR);
System.clearProperty(PropertiesFactory.APOLLO_PROPERTY_ORDER_ENABLE);
System.clearProperty(ApolloClientSystemConsts.APOLLO_PROPERTY_NAMES_CACHE_ENABLE);
System.clearProperty(ApolloClientSystemConsts.APOLLO_CACHE_KUBERNETES_NAMESPACE);
System.clearProperty(ApolloClientSystemConsts.APOLLO_KUBERNETES_CACHE_ENABLE);
}

@Test
@@ -243,6 +245,35 @@ public void testDefaultLocalCacheDir() throws Exception {
assertEquals("/opt/data/" + someAppId, configUtil.getDefaultLocalCacheDir());
}

@Test
public void testK8sNamespaceWithSystemProperty() {
String someK8sNamespace = "someK8sNamespace";

System.setProperty(ApolloClientSystemConsts.APOLLO_CACHE_KUBERNETES_NAMESPACE, someK8sNamespace);

ConfigUtil configUtil = new ConfigUtil();

assertEquals(someK8sNamespace, configUtil.getK8sNamespace());
}

@Test
public void testK8sNamespaceWithDefault() {
ConfigUtil configUtil = new ConfigUtil();

assertEquals(ConfigConsts.KUBERNETES_CACHE_CONFIG_MAP_NAMESPACE_DEFAULT, configUtil.getK8sNamespace());
}

@Test
public void testKubernetesCacheEnabledWithSystemProperty() {
boolean someKubernetesCacheEnabled = true;

System.setProperty(ApolloClientSystemConsts.APOLLO_KUBERNETES_CACHE_ENABLE, String.valueOf(someKubernetesCacheEnabled));

ConfigUtil configUtil = new ConfigUtil();

assertTrue(configUtil.isPropertyKubernetesCacheEnabled());
}

@Test
public void testCustomizePropertiesOrdered() {
boolean propertiesOrdered = true;
Original file line number Diff line number Diff line change
@@ -73,6 +73,16 @@ public class ApolloClientSystemConsts {
@Deprecated
public static final String DEPRECATED_APOLLO_CACHE_DIR_ENVIRONMENT_VARIABLES = "APOLLO_CACHEDIR";

/**
* kubernetes configmap cache namespace
*/
public static final String APOLLO_CACHE_KUBERNETES_NAMESPACE = "apollo.cache.kubernetes.namespace";

/**
* kubernetes configmap cache namespace environment variables
*/
public static final String APOLLO_CACHE_KUBERNETES_NAMESPACE_ENVIRONMENT_VARIABLES = "APOLLO_CACHE_KUBERNETES_NAMESPACE";

/**
* apollo client access key
*/
@@ -157,6 +167,16 @@ public class ApolloClientSystemConsts {
*/
public static final String APOLLO_CACHE_FILE_ENABLE_ENVIRONMENT_VARIABLES = "APOLLO_CACHE_FILE_ENABLE";

/**
* enable property names cache
*/
public static final String APOLLO_KUBERNETES_CACHE_ENABLE = "apollo.cache.kubernetes.enable";

/**
* enable property names cache environment variables
*/
public static final String APOLLO_KUBERNETES_CACHE_ENABLE_ENVIRONMENT_VARIABLES = "APOLLO_KUBERNETES_CACHE_ENABLE";

/**
* enable apollo overrideSystemProperties
*/
Original file line number Diff line number Diff line change
@@ -20,10 +20,12 @@ public interface ConfigConsts {
String NAMESPACE_APPLICATION = "application";
String CLUSTER_NAME_DEFAULT = "default";
String CLUSTER_NAMESPACE_SEPARATOR = "+";
String APOLLO_CONFIG_CACHE = "apollo-configcache-";
String APOLLO_CLUSTER_KEY = "apollo.cluster";
String APOLLO_META_KEY = "apollo.meta";
String CONFIG_FILE_CONTENT_KEY = "content";
String NO_APPID_PLACEHOLDER = "ApolloNoAppIdPlaceHolder";
String KUBERNETES_CACHE_CONFIG_MAP_NAMESPACE_DEFAULT = "default";
String APOLLO_AUTO_UPDATE_INJECTED_SPRING_PROPERTIES = "ApolloAutoUpdateInjectedSpringProperties";
long NOTIFICATION_ID_PLACEHOLDER = -1;
}
6 changes: 6 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
@@ -164,6 +164,12 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>io.kubernetes</groupId>
<artifactId>client-java</artifactId>
<version>18.0.0</version>
<optional>true</optional>
</dependency>
</dependencies>
</dependencyManagement>

0 comments on commit 67eb866

Please sign in to comment.