Skip to content

Commit

Permalink
ST: Fix for ConnectS2IST and fix for Docker Tags (strimzi#1287)
Browse files Browse the repository at this point in the history 
1. Fixed issue with Kafka version in image Tags during CO deployment process
2. Removed redundant checks in ConnectS2IST
  • Loading branch information
@serrss
serrss authored Feb 26, 2019
1 parent 97ccb86 commit 69c9c14
Show file tree
Hide file tree
Showing 5 changed files with 19 additions and 16 deletions.
3 changes: 3 additions & 0 deletions HACKING.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,3 +223,6 @@ Ex)
### Log level

To set the log level of Strimzi for system tests need to add system property `TEST_STRIMZI_LOG_LEVEL` with one of the following values: `ERROR`, `WARNING`, `INFO`, `DEBUG`, `TRACE`.

### Execute ST with custom Kafka version
To set custom Kafka version in system tests need to add system property `ST_KAFKA_VERSION` with one of the following values: `2.0.0`, `2.0.1`, `2.1.0`
13 changes: 9 additions & 4 deletions systemtest/src/main/java/io/strimzi/systemtest/Resources.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@
import io.fabric8.kubernetes.api.model.Doneable;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.KubernetesResourceList;
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder;
import io.fabric8.kubernetes.api.model.apps.DeploymentList;
import io.fabric8.kubernetes.api.model.apps.DoneableDeployment;
import io.fabric8.kubernetes.api.model.batch.Job;
import io.fabric8.kubernetes.api.model.KubernetesResourceList;
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.fabric8.kubernetes.api.model.rbac.DoneableKubernetesClusterRoleBinding;
import io.fabric8.kubernetes.api.model.rbac.DoneableKubernetesRoleBinding;
import io.fabric8.kubernetes.api.model.rbac.KubernetesClusterRoleBinding;
Expand Down Expand Up @@ -76,6 +76,7 @@ public class Resources {
private static final long TIMEOUT_FOR_DEPLOYMENT_CONFIG_READINESS = Duration.ofMinutes(7).toMillis();
private static final long TIMEOUT_FOR_RESOURCE_CREATION = Duration.ofMinutes(5).toMillis();
public static final long TIMEOUT_FOR_RESOURCE_READINESS = Duration.ofMinutes(7).toMillis();
private static final String KAFKA_VERSION = System.getenv().getOrDefault("ST_KAFKA_VERSION", "2.1.0");

public static final String STRIMZI_PATH_TO_CO_CONFIG = "../install/cluster-operator/050-Deployment-strimzi-cluster-operator.yaml";
public static final String STRIMZI_DEPLOYMENT_NAME = "strimzi-cluster-operator";
Expand Down Expand Up @@ -245,6 +246,7 @@ public KafkaBuilder defaultKafka(String name, int kafkaReplicas) {
.withMetadata(new ObjectMetaBuilder().withName(name).withNamespace(client().getNamespace()).build())
.withNewSpec()
.withNewKafka()
.withVersion(KAFKA_VERSION)
.withReplicas(kafkaReplicas)
.withNewEphemeralStorage().endEphemeralStorage()
.addToConfig("offsets.topic.replication.factor", Math.min(kafkaReplicas, 3))
Expand Down Expand Up @@ -323,6 +325,7 @@ private KafkaConnectBuilder defaultKafkaConnect(String name, int kafkaConnectRep
return new KafkaConnectBuilder()
.withMetadata(new ObjectMetaBuilder().withName(name).withNamespace(client().getNamespace()).build())
.withNewSpec()
.withVersion(KAFKA_VERSION)
.withBootstrapServers(KafkaResources.plainBootstrapAddress(name))
.withReplicas(kafkaConnectReplicas)
.withNewResources()
Expand Down Expand Up @@ -369,6 +372,7 @@ private KafkaConnectS2IBuilder defaultKafkaConnectS2I(String name, int kafkaConn
return new KafkaConnectS2IBuilder()
.withMetadata(new ObjectMetaBuilder().withName(name).withNamespace(client().getNamespace()).build())
.withNewSpec()
.withVersion(KAFKA_VERSION)
.withBootstrapServers(KafkaResources.plainBootstrapAddress(name))
.withReplicas(kafkaConnectS2IReplicas)
.endSpec();
Expand Down Expand Up @@ -403,6 +407,7 @@ private KafkaMirrorMakerBuilder defaultMirrorMaker(String name, String sourceBoo
return new KafkaMirrorMakerBuilder()
.withMetadata(new ObjectMetaBuilder().withName(name).withNamespace(client().getNamespace()).build())
.withNewSpec()
.withVersion(KAFKA_VERSION)
.withNewConsumer()
.withBootstrapServers(tlsListener ? sourceBootstrapServer + "-kafka-bootstrap:9093" : sourceBootstrapServer + "-kafka-bootstrap:9092")
.withGroupId(groupId)
Expand Down Expand Up @@ -617,10 +622,10 @@ Deployment getDeploymentFromYaml(String yamlPath) {
}

DoneableDeployment clusterOperator(String namespace) {
return clusterOperator(defaultCLusterOperator(namespace).build());
return clusterOperator(defaultClusterOperator(namespace).build());
}

DeploymentBuilder defaultCLusterOperator(String namespace) {
DeploymentBuilder defaultClusterOperator(String namespace) {

Deployment clusterOperator = getDeploymentFromYaml(STRIMZI_PATH_TO_CO_CONFIG);

Expand Down
2 changes: 1 addition & 1 deletion systemtest/src/main/java/io/strimzi/systemtest/utils/StUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ public static Map<String, String> waitTillDepHasRolled(KubernetesClient client,
}

public static File downloadAndUnzip(String url) throws IOException {
InputStream bais = (InputStream) URI.create(url).toURL().getContent();
InputStream bais = (InputStream) URI.create(url).toURL().openConnection().getContent();
File dir = Files.createTempDirectory(StUtils.class.getName()).toFile();
dir.deleteOnExit();
ZipInputStream zin = new ZipInputStream(bais);
Expand Down
7 changes: 2 additions & 5 deletions systemtest/src/test/java/io/strimzi/systemtest/ConnectS2IST.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,23 +37,20 @@ class ConnectS2IST extends AbstractST {
@OpenShiftOnly
@Tag(FLAKY)
void testDeployS2IWithMongoDBPlugin() throws IOException {
resources().kafkaConnectS2I(CONNECT_CLUSTER_NAME, 1)
testClassResources.kafkaConnectS2I(CONNECT_CLUSTER_NAME, 1)
.editMetadata()
.addToLabels("type", "kafka-connect-s2i")
.endMetadata()
.done();

File dir = StUtils.downloadAndUnzip("https://repo1.maven.org/maven2/io/debezium/debezium-connector-mongodb/0.3.0/debezium-connector-mongodb-0.3.0-plugin.zip");

String connectS2IPodName = KUBE_CLIENT.listResourcesByLabel("pod", "type=kafka-connect-s2i").get(0);

// Start a new image build using the plugins directory
KUBE_CLIENT.exec("oc", "start-build", CONNECT_DEPLOYMENT_NAME, "--from-dir", dir.getAbsolutePath());
KUBE_CLIENT.waitForResourceDeletion("pod", connectS2IPodName);

KUBE_CLIENT.waitForDeploymentConfig(CONNECT_DEPLOYMENT_NAME);

connectS2IPodName = KUBE_CLIENT.listResourcesByLabel("pod", "type=kafka-connect-s2i").get(0);
String connectS2IPodName = KUBE_CLIENT.listResourcesByLabel("pod", "type=kafka-connect-s2i").get(0);
String plugins = KUBE_CLIENT.execInPod(connectS2IPodName, "curl", "-X", "GET", "http://localhost:8083/connector-plugins").out();

assertThat(plugins, containsString("io.debezium.connector.mongodb.MongoDbConnector"));
Expand Down
10 changes: 4 additions & 6 deletions test/src/main/java/io/strimzi/test/TestUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public final class TestUtils {

public static final String CRD_KAFKA_MIRROR_MAKER = "../install/cluster-operator/045-Crd-kafkamirrormaker.yaml";

private static final Pattern KAFKA_COMPONENT_PATTERN = Pattern.compile(":([^:]*?)-kafka-([0-9.]+)$");
private static final Pattern KAFKA_COMPONENT_PATTERN = Pattern.compile(":([^:]*?)-kafka-(?<version>[0-9.])");
private static final Pattern VERSION_IMAGE_PATTERN = Pattern.compile("(?<version>[0-9.]+)=(?<image>[^\\s]*)");

private TestUtils() {
Expand Down Expand Up @@ -147,12 +147,12 @@ public static String getFileAsString(String filePath) {
return "";
}

public static String changeOrgAndTag(String image, String newOrg, String newTag, String kafkaVersion) {
public static String changeOrgAndTag(String image, String newOrg, String newTag) {
image = image.replaceFirst("^strimzi/", newOrg + "/");
Matcher m = KAFKA_COMPONENT_PATTERN.matcher(image);
StringBuffer sb = new StringBuffer();
if (m.find()) {
m.appendReplacement(sb, ":" + newTag + "-kafka-" + kafkaVersion);
m.appendReplacement(sb, ":" + newTag + "-kafka-" + m.group("version"));
m.appendTail(sb);
image = sb.toString();
} else {
Expand All @@ -164,11 +164,9 @@ public static String changeOrgAndTag(String image, String newOrg, String newTag,
public static String changeOrgAndTag(String image) {
String strimziOrg = "strimzi";
String strimziTag = "latest";
String kafkaVersion = "2.1.0";
String dockerOrg = System.getenv().getOrDefault("DOCKER_ORG", strimziOrg);
String dockerTag = System.getenv().getOrDefault("DOCKER_TAG", strimziTag);
kafkaVersion = System.getenv().getOrDefault("KAFKA_VERSION", kafkaVersion);
return changeOrgAndTag(image, dockerOrg, dockerTag, kafkaVersion);
return changeOrgAndTag(image, dockerOrg, dockerTag);
}

public static String changeOrgAndTagInImageMap(String imageMap) {
Expand Down

0 comments on commit 69c9c14

Please sign in to comment.