fix(discovery): check container discovery domain socket readability b…

…efore start (#251) * fix(discovery): check container discovery domain socket readability before start * enable docker discovery in smoketest * configurable container discovery poll period and request timeout
cryostatio · Jan 22, 2024 · 29e7abe · 29e7abe
1 parent a54e7f3
commit 29e7abe
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 4 deletions.
diff --git a/smoketest/compose/cryostat.yml b/smoketest/compose/cryostat.yml
@@ -28,8 +28,9 @@ services:
       QUARKUS_HTTP_HOST: "cryostat"
       QUARKUS_HTTP_PORT: ${CRYOSTAT_HTTP_PORT}
       QUARKUS_HIBERNATE_ORM_LOG_SQL: "true"
-      CRYOSTAT_DISCOVERY_PODMAN_ENABLED: "true"
       CRYOSTAT_DISCOVERY_JDP_ENABLED: "true"
+      CRYOSTAT_DISCOVERY_PODMAN_ENABLED: "true"
+      CRYOSTAT_DISCOVERY_DOCKER_ENABLED: "true"
       JAVA_OPTS_APPEND: "-XX:+FlightRecorder -XX:StartFlightRecording=name=onstart,settings=default,disk=true,maxage=5m -XX:StartFlightRecording=name=startup,settings=profile,disk=true,duration=30s -Dcom.sun.management.jmxremote.autodiscovery=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9091 -Dcom.sun.management.jmxremote.rmi.port=9091 -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false"
     restart: unless-stopped
     healthcheck:

diff --git a/src/main/java/io/cryostat/ConfigProperties.java b/src/main/java/io/cryostat/ConfigProperties.java
@@ -20,6 +20,10 @@ public class ConfigProperties {
     public static final String AWS_OBJECT_EXPIRATION_LABELS =
             "storage.buckets.archives.expiration-label";
 
+    public static final String CONTAINERS_POLL_PERIOD = "cryostat.discovery.containers.poll-period";
+    public static final String CONTAINERS_REQUEST_TIMEOUT =
+            "cryostat.discovery.containers.request-timeout";
+
     public static final String CONNECTIONS_MAX_OPEN = "cryostat.connections.max-open";
     public static final String CONNECTIONS_TTL = "cryostat.connections.ttl";
     public static final String CONNECTIONS_FAILED_BACKOFF = "cryostat.connections.failed-backoff";

diff --git a/src/main/java/io/cryostat/discovery/ContainerDiscovery.java b/src/main/java/io/cryostat/discovery/ContainerDiscovery.java
@@ -18,6 +18,8 @@
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -34,8 +36,10 @@
 
 import javax.management.remote.JMXServiceURL;
 
+import io.cryostat.ConfigProperties;
 import io.cryostat.URIUtil;
 import io.cryostat.core.net.JFRConnectionToolkit;
+import io.cryostat.core.sys.FileSystem;
 import io.cryostat.targets.Target;
 import io.cryostat.targets.Target.Annotations;
 import io.cryostat.targets.Target.EventKind;
@@ -131,11 +135,18 @@ public abstract class ContainerDiscovery {
     public static final String JMX_PORT_LABEL = "io.cryostat.jmxPort";
 
     @Inject Logger logger;
+    @Inject FileSystem fs;
     @Inject Vertx vertx;
     @Inject WebClient webClient;
     @Inject JFRConnectionToolkit connectionToolkit;
     @Inject ObjectMapper mapper;
 
+    @ConfigProperty(name = ConfigProperties.CONTAINERS_POLL_PERIOD)
+    Duration pollPeriod;
+
+    @ConfigProperty(name = ConfigProperties.CONTAINERS_REQUEST_TIMEOUT)
+    Duration requestTimeout;
+
     protected long timerId;
 
     protected final CopyOnWriteArrayList<ContainerSpec> containers = new CopyOnWriteArrayList<>();
@@ -146,6 +157,15 @@ void onStart(@Observes StartupEvent evt) {
             return;
         }
 
+        Path socketPath = Path.of(getSocket().path());
+        if (!(fs.exists(socketPath) && fs.isReadable(socketPath))) {
+            logger.errorv(
+                    "{0} enabled but socket {1} is not accessible!",
+                    getClass().getName(), socketPath);
+            return;
+        }
+        logger.infov("{0} started", getClass().getName());
+
         DiscoveryNode universe = DiscoveryNode.getUniverse();
         if (DiscoveryNode.getRealm(getRealm()).isEmpty()) {
             DiscoveryPlugin plugin = new DiscoveryPlugin();
@@ -158,7 +178,7 @@ void onStart(@Observes StartupEvent evt) {
         }
 
         queryContainers();
-        this.timerId = vertx.setPeriodic(10_000, unused -> queryContainers());
+        this.timerId = vertx.setPeriodic(pollPeriod.toMillis(), unused -> queryContainers());
     }
 
     void onStop(@Observes ShutdownEvent evt) {
@@ -219,7 +239,7 @@ private void doContainerListRequest(Consumer<List<ContainerSpec>> successHandler
                     .addQueryParam(
                             "filters",
                             mapper.writeValueAsString(Map.of("label", List.of(DISCOVERY_LABEL))))
-                    .timeout(2_000L)
+                    .timeout(requestTimeout.toMillis())
                     .as(BodyCodec.string())
                     .send()
                     .subscribe()
@@ -249,7 +269,7 @@ private CompletableFuture<ContainerDetails> doContainerInspectRequest(ContainerS
         URI requestPath = URI.create(getContainerQueryURL(container));
         webClient
                 .request(HttpMethod.GET, getSocket(), 80, "localhost", requestPath.toString())
-                .timeout(2_000L)
+                .timeout(requestTimeout.toMillis())
                 .as(BodyCodec.string())
                 .send()
                 .subscribe()

diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
@@ -1,5 +1,7 @@
 quarkus.naming.enable-jndi=true
 cryostat.discovery.jdp.enabled=false
+cryostat.discovery.containers.poll-period=10s
+cryostat.discovery.containers.request-timeout=2s
 cryostat.discovery.podman.enabled=false
 cryostat.discovery.docker.enabled=false
 quarkus.test.integration-test-profile=test