ADH-5525: Prepare product tests environment for ADB plugin

core/docker/arenadata/docker-compose-test-trino-adb-distributed.yaml

@@ -0,0 +1,149 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3.8"
+services:
+    adb:
+        image: "hub.adsw.io/trino/gpdb6-distributed:it"
+        restart: on-failure:3
+        working_dir: /home/gpadmin
+        hostname: mdw
+        ports:
+            - "6000:6000"
+        environment:
+            - DOCKER_GP_CLUSTER_HOSTS=mdw,smdw,sdw1,sdw2
+            - DOCKER_GP_MASTER_SERVER=mdw
+            - DOCKER_GP_STANDBY_SERVER=smdw
+            - DOCKER_GP_SEGMENT_SERVERS=sdw1,sdw2
+            - DOCKER_GP_PRIMARY_SEGMENTS_PER_HOST=2
+            - DOCKER_GP_WITH_MIRROR=true
+        depends_on:
+            - sdw1
+            - sdw2
+            - smdw
+        healthcheck:
+            test: 'bash /home/gpadmin/healthcheck.sh true 2 true'
+            interval: 10s
+            timeout: 1s
+            retries: 20
+        entrypoint: ["bash", "start_adb.sh" ]
+        privileged: true
+        extra_hosts:
+            - "host.docker.internal:host-gateway"
+        sysctls:
+            kernel.sem: 500 1024000 200 4096
+
+    sdw1:
+        image: "hub.adsw.io/trino/gpdb6-distributed:it"
+        restart: on-failure:3
+        privileged: true
+        hostname: sdw1
+        healthcheck:
+            test: 'bash /home/gpadmin/healthcheck.sh false 2 true'
+        environment:
+            - DOCKER_GP_CLUSTER_HOSTS=mdw,smdw,sdw1,sdw2
+            - DOCKER_GP_MASTER_SERVER=mdw
+            - DOCKER_GP_STANDBY_SERVER=smdw
+            - DOCKER_GP_SEGMENT_SERVERS=sdw1,sdw2
+            - DOCKER_GP_PRIMARY_SEGMENTS_PER_HOST=2
+            - DOCKER_GP_WITH_MIRROR=true
+        entrypoint: ["bash", "start_adb.sh" ]
+        extra_hosts:
+            - "host.docker.internal:host-gateway"
+        sysctls:
+            kernel.sem: 500 1024000 200 4096
+
+    sdw2:
+        image: "hub.adsw.io/trino/gpdb6-distributed:it"
+        restart: on-failure:3
+        privileged: true
+        hostname: sdw2
+        healthcheck:
+            test: 'bash /home/gpadmin/healthcheck.sh false 2 true'
+        environment:
+            - DOCKER_GP_CLUSTER_HOSTS=mdw,smdw,sdw1,sdw2
+            - DOCKER_GP_MASTER_SERVER=mdw
+            - DOCKER_GP_STANDBY_SERVER=smdw
+            - DOCKER_GP_SEGMENT_SERVERS=sdw1,sdw2
+            - DOCKER_GP_PRIMARY_SEGMENTS_PER_HOST=2
+            - DOCKER_GP_WITH_MIRROR=true
+        entrypoint: ["bash", "start_adb.sh" ]
+        extra_hosts:
+            - "host.docker.internal:host-gateway"
+        sysctls:
+            kernel.sem: 500 1024000 200 4096
+
+    smdw:
+        image: "hub.adsw.io/trino/gpdb6-distributed:it"
+        restart: on-failure:3
+        privileged: true
+        hostname: smdw
+        healthcheck:
+            test: 'bash /home/gpadmin/healthcheck.sh true 2 true'
+        environment:
+            - DOCKER_GP_CLUSTER_HOSTS=mdw,smdw,sdw1,sdw2
+            - DOCKER_GP_MASTER_SERVER=mdw
+            - DOCKER_GP_STANDBY_SERVER=smdw
+            - DOCKER_GP_SEGMENT_SERVERS=sdw1,sdw2
+            - DOCKER_GP_PRIMARY_SEGMENTS_PER_HOST=2
+            - DOCKER_GP_WITH_MIRROR=true
+        entrypoint: ["bash", "start_adb.sh" ]
+        extra_hosts:
+            - "host.docker.internal:host-gateway"
+        sysctls:
+            kernel.sem: 500 1024000 200 4096
+
+    postgres:
+        image: "docker.io/library/postgres:12.0"
+        restart: unless-stopped
+        container_name: postgres-db
+        hostname: postgres-db
+        environment:
+            POSTGRES_DB: '${POSTGRES_DB}'
+            POSTGRES_USER: '${POSTGRES_USER}'
+            POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
+        ports:
+            - '5432:5432'
+        healthcheck:
+            test: psql -d ${POSTGRES_DB} -U ${POSTGRES_USER} -Atc 'SELECT 1;'
+            interval: 30s
+            timeout: 15s
+            retries: 3
+
+    coordinator:
+        image: "trino:${TRINO_VERSION}"
+        ports:
+            - "8080:8080"
+            - "8990:8990"
+        container_name: "coordinator"
+        volumes:
+            - ./coordinator/etc:/etc/trino:rw
+
+    worker0:
+        image: "trino:${TRINO_VERSION}"
+        container_name: "worker0"
+        ports:
+            - "8081:8080"
+            - "8991:8990"
+        volumes:
+            - ./worker/etc:/etc/trino:rw
+
+    worker1:
+        image: "trino:${TRINO_VERSION}"
+        container_name: "worker1"
+        ports:
+            - "8082:8080"
+            - "8992:8990"
+        volumes:
+            - ./worker/etc:/etc/trino:rw

testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/DockerContainer.java

@@ -68,7 +68,7 @@ public class DockerContainer
         extends FixedHostPortGenericContainer<DockerContainer>
 {
     private static final Logger log = Logger.get(DockerContainer.class);
-    private static final long NANOSECONDS_PER_SECOND = 1_000 * 1_000 * 1_000L;
+    public static final long NANOSECONDS_PER_SECOND = 1_000 * 1_000 * 1_000L;
 
     private static final Timeout<ExecResult> asyncTimeout = Timeout.<ExecResult>builder(ofSeconds(30))
             .withInterrupt()

testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAdb.java

@@ -0,0 +1,127 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.trino.tests.product.launcher.env.environment;
+
+import com.github.dockerjava.api.model.HealthCheck;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+import io.trino.tests.product.launcher.docker.DockerFiles;
+import io.trino.tests.product.launcher.env.DockerContainer;
+import io.trino.tests.product.launcher.env.Environment;
+import io.trino.tests.product.launcher.env.EnvironmentProvider;
+import io.trino.tests.product.launcher.env.common.StandardMultinode;
+import io.trino.tests.product.launcher.env.common.TestsEnvironment;
+import io.trino.tests.product.launcher.testcontainers.PortBinder;
+import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+
+import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts;
+import static io.trino.tests.product.launcher.env.DockerContainer.NANOSECONDS_PER_SECOND;
+import static java.util.Objects.requireNonNull;
+import static org.testcontainers.containers.wait.strategy.Wait.forHealthcheck;
+import static org.testcontainers.utility.MountableFile.forHostPath;
+
+@TestsEnvironment
+public final class EnvMultinodeAdb
+        extends EnvironmentProvider
+{
+    public static final String TRINO = "trino";
+    public static final int POSTGRESQL_PORT = 5432;
+    public static final int ADB_MASTER_PORT = 6000;
+    public static final String ADB_MASTER_NAME = "mdw";
+    public static final String ADB_SEGMENT1_NAME = "sdw1";
+    public static final String ADB_SEGMENT2_NAME = "sdw2";
+    public static final String ADB_STANDBY_NAME = "smdw";
+    public static final Map<String, String> SYSCTLS = Map.of("kernel.sem", "500 1024000 200 4096");
+
+    private final DockerFiles dockerFiles;
+    private final PortBinder portBinder;
+
+    @Inject
+    public EnvMultinodeAdb(StandardMultinode standardMultinode, DockerFiles dockerFiles, PortBinder portBinder)
+    {
+        super(ImmutableList.of(standardMultinode));
+        this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null");
+        this.portBinder = requireNonNull(portBinder, "portBinder is null");
+    }
+
+    @Override
+    public void extendEnvironment(Environment.Builder builder)
+    {
+        builder.addConnector(
+                "adb",
+                forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-adb/adb.properties")));
+        builder.addContainer(createPostgreSql());
+        builder.addContainers(
+                createAdb(ADB_MASTER_NAME),
+                createAdb(ADB_STANDBY_NAME),
+                createAdb(ADB_SEGMENT1_NAME),
+                createAdb(ADB_SEGMENT2_NAME));
+    }
+
+    private DockerContainer createPostgreSql()
+    {
+        DockerContainer container = new DockerContainer("postgres:12", "postgres-db")
+                .withEnv("POSTGRES_PASSWORD", TRINO)
+                .withEnv("POSTGRES_USER", TRINO)
+                .withEnv("POSTGRES_DB", TRINO)
+                .withEnv("PGPORT", Integer.toString(POSTGRESQL_PORT))
+                .withStartupCheckStrategy(new IsRunningStartupCheckStrategy())
+                .waitingFor(forSelectedPorts(POSTGRESQL_PORT));
+        portBinder.exposePort(container, POSTGRESQL_PORT);
+        return container;
+    }
+
+    private DockerContainer createAdb(String name)
+    {
+        DockerContainer container = new DockerContainer("hub.adsw.io/trino/gpdb6-distributed:it", name)
+                .withWorkingDirectory("/home/gpadmin")
+                .withEnv("DOCKER_GP_CLUSTER_HOSTS",
+                        String.join(",", ADB_MASTER_NAME, ADB_STANDBY_NAME, ADB_SEGMENT1_NAME, ADB_SEGMENT2_NAME))
+                .withEnv("DOCKER_GP_MASTER_SERVER", ADB_MASTER_NAME)
+                .withEnv("DOCKER_GP_STANDBY_SERVER", ADB_STANDBY_NAME)
+                .withEnv("DOCKER_GP_SEGMENT_SERVERS", String.join(",", ADB_SEGMENT1_NAME, ADB_SEGMENT2_NAME))
+                .withEnv("DOCKER_GP_PRIMARY_SEGMENTS_PER_HOST", "2")
+                .withEnv("DOCKER_GP_WITH_MIRROR", "true")
+                .withPrivilegedMode(true)
+                .withCommand("bash", "start_adb.sh")
+                .withStartupCheckStrategy(new IsRunningStartupCheckStrategy())
+                .waitingFor(forHealthcheck().withStartupTimeout(Duration.ofSeconds(180)))
+                .withCreateContainerCmdModifier(cmd -> {
+                    HealthCheck healthCheck = new HealthCheck()
+                            .withTest(List.of("CMD",
+                                    "bash",
+                                    "/home/gpadmin/healthcheck.sh",
+                                    String.valueOf(ADB_MASTER_NAME.equals(name) || ADB_STANDBY_NAME.equals(name)),
+                                    "2",
+                                    "true"))
+                            .withInterval(NANOSECONDS_PER_SECOND * 10)
+                            .withTimeout(NANOSECONDS_PER_SECOND)
+                            .withRetries(20);
+                    cmd.withHealthcheck(healthCheck)
+                            .getHostConfig()
+                            .withExtraHosts("host.docker.internal:host-gateway")
+                            .withSysctls(SYSCTLS)
+                            .withAutoRemove(false);
+                });
+        if (ADB_MASTER_NAME.equals(name)) {
+            portBinder.exposePort(container, ADB_MASTER_PORT);
+        }
+        return container;
+    }
+}

testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-adb/adb.properties

@@ -0,0 +1,4 @@
+connector.name=adb
+connection-url=jdbc:postgresql://mdw:6000/adb
+connection-user=gpadmin
+connection-password=gpadmin

testing/trino-product-tests/src/main/resources/tempto-configuration.yaml

@@ -198,6 +198,13 @@ databases:
     jdbc_password: exasol
     jdbc_pooling: true
     table_manager_type: jdbc
+  adb:
+    jdbc_driver_class: org.postgresql.Driver
+    jdbc_url: jdbc:postgresql://mdw:6000/adb
+    jdbc_user: gpadmin
+    jdbc_password: gpadmin
+    jdbc_pooling: true
+    table_manager_type: jdbc
 
 tests:
   hdfs: