diff --git a/pom.xml b/pom.xml
index 8f630a2f0..605dba7c6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -70,6 +70,8 @@
     <!-- https://www.jenkins.io/doc/developer/plugin-development/choosing-jenkins-baseline/ -->
     <jenkins.baseline>2.479</jenkins.baseline>
     <jenkins.version>${jenkins.baseline}.1</jenkins.version>
+    <!-- TODO until in parent -->
+    <jenkins-test-harness.version>2403.v256947ecb_c8a_</jenkins-test-harness.version>
     <gitHubRepo>jenkinsci/${project.artifactId}-plugin</gitHubRepo>
     <spotless.check.skip>false</spotless.check.skip>
   </properties>
diff --git a/src/main/java/com/cloudbees/jenkins/support/SupportLogHandler.java b/src/main/java/com/cloudbees/jenkins/support/SupportLogHandler.java
index 842b34d6e..fde60d5c4 100644
--- a/src/main/java/com/cloudbees/jenkins/support/SupportLogHandler.java
+++ b/src/main/java/com/cloudbees/jenkins/support/SupportLogHandler.java
@@ -234,6 +234,7 @@ private void setFile(File file) throws FileNotFoundException {
                 parentFile.mkdirs();
             }
 
+            StreamUtils.closeQuietly(null); // ensure class is loaded so close() can succeed
             boolean success = false;
             FileOutputStream fos = null;
             BufferedOutputStream bos = null;
diff --git a/src/main/java/com/cloudbees/jenkins/support/SupportPlugin.java b/src/main/java/com/cloudbees/jenkins/support/SupportPlugin.java
index 3e57d01a3..bf8256f1b 100644
--- a/src/main/java/com/cloudbees/jenkins/support/SupportPlugin.java
+++ b/src/main/java/com/cloudbees/jenkins/support/SupportPlugin.java
@@ -862,6 +862,15 @@ public Void call() {
             // avoid double installation of the handler. JNLP agents can reconnect to the controller multiple times
             // and each connection gets a different RemoteClassLoader, so we need to evict them by class name,
             // not by their identity.
+            closeAll();
+            Runtime.getRuntime().addShutdownHook(new Thread(LogInitializer::closeAll, "close log handlers"));
+            LogHolder.AGENT_LOG_HANDLER.setLevel(level);
+            LogHolder.AGENT_LOG_HANDLER.setDirectory(new File(rootPath.getRemote(), SUPPORT_DIRECTORY_NAME), "all");
+            ROOT_LOGGER.addHandler(LogHolder.AGENT_LOG_HANDLER);
+            return null;
+        }
+
+        private static void closeAll() {
             for (Handler h : ROOT_LOGGER.getHandlers()) {
                 if (h.getClass()
                         .getName()
@@ -874,10 +883,6 @@ public Void call() {
                     }
                 }
             }
-            LogHolder.AGENT_LOG_HANDLER.setLevel(level);
-            LogHolder.AGENT_LOG_HANDLER.setDirectory(new File(rootPath.getRemote(), SUPPORT_DIRECTORY_NAME), "all");
-            ROOT_LOGGER.addHandler(LogHolder.AGENT_LOG_HANDLER);
-            return null;
         }
     }
 
diff --git a/src/test/java/com/cloudbees/jenkins/support/impl/SmartLogCleanerTest.java b/src/test/java/com/cloudbees/jenkins/support/impl/SmartLogCleanerTest.java
index e330cc6cc..8f6afc190 100644
--- a/src/test/java/com/cloudbees/jenkins/support/impl/SmartLogCleanerTest.java
+++ b/src/test/java/com/cloudbees/jenkins/support/impl/SmartLogCleanerTest.java
@@ -49,6 +49,7 @@ public void cleanUp() throws Exception {
         }
 
         assertEquals(cacheDir.list().length, 2);
+        agent2.toComputer().disconnect(null).get();
         j.getInstance().removeNode(agent2);
 
         generateBundle();