diff --git a/.github/workflows/integ-test.yml b/.github/workflows/integ-test.yml
index 370cad91..0ee2d101 100644
--- a/.github/workflows/integ-test.yml
+++ b/.github/workflows/integ-test.yml
@@ -49,6 +49,13 @@ jobs:
           echo "JAVA21_HOME=$JAVA_HOME" >> $GITHUB_ENV
           echo "BUILD_JAVA_HOME=$JAVA_HOME" >> $GITHUB_ENV
 
+      - name: Install JDK 21
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'adopt'
+          java-version: '21'
+      - run: echo "JAVA21_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
       - name: Install JDK 17
         uses: actions/setup-java@v3
         with:
diff --git a/Makefile b/Makefile
index a1d1ae7c..f55e885e 100644
--- a/Makefile
+++ b/Makefile
@@ -39,7 +39,7 @@ check-pip:
 	@if ! $(PIP) > /dev/null 2>&1 || ! $(PIP) install pip > /dev/null 2>&1; then make pyinst38; fi
 
 check-java:
-	@if ! test "$(JAVA_HOME)" || ! java --version > /dev/null 2>&1 || ! javadoc --help > /dev/null 2>&1; then \
+	@if ! test "$(JAVA21_HOME)" || ! java --version > /dev/null 2>&1 || ! javadoc --help > /dev/null 2>&1; then \
 	    echo "Java installation issues for running integration tests" >&2; \
 	    exit 1; \
 	fi
diff --git a/osbenchmark/builder/provisioner.py b/osbenchmark/builder/provisioner.py
index ee878850..4744e173 100644
--- a/osbenchmark/builder/provisioner.py
+++ b/osbenchmark/builder/provisioner.py
@@ -346,7 +346,7 @@ def install(self, os_home_path, plugin_url=None):
             self.logger.info("Installing [%s] into [%s]", self.plugin_name, os_home_path)
             install_cmd = '%s install --batch "%s"' % (installer_binary_path, self.plugin_name)
 
-        return_code = process.run_subprocess_with_logging(install_cmd, env=self.env())
+        output, return_code = process.run_subprocess_with_logging(install_cmd, env=self.env(), capture_output=True)
         if return_code == 0:
             self.logger.info("Successfully installed [%s].", self.plugin_name)
         elif return_code == 64:
@@ -356,9 +356,9 @@ def install(self, os_home_path, plugin_url=None):
             raise exceptions.SupplyError("I/O error while trying to install [%s]" % self.plugin_name)
         else:
             raise exceptions.BenchmarkError(
-                "Unknown error while trying to install [%s] (installer return code [%s]). "
+                "Unknown error '%s' while trying to install [%s] (installer return code [%s]). "
                 "Please check the logs." %
-                                        (self.plugin_name, str(return_code)))
+                                        (output, self.plugin_name, str(return_code)))
 
     def invoke_install_hook(self, phase, variables):
         self.hook_handler.invoke(phase.name, variables=variables, env=self.env())
diff --git a/osbenchmark/builder/supplier.py b/osbenchmark/builder/supplier.py
index 473ab229..3b7dac1c 100644
--- a/osbenchmark/builder/supplier.py
+++ b/osbenchmark/builder/supplier.py
@@ -702,7 +702,7 @@ def run(self, command, override_src_dir=None):
         log_file = os.path.join(self.log_dir, "build.log")
 
         # we capture all output to a dedicated build log file
-        build_cmd = "export JAVA_HOME={}; cd {}; {} > {} 2>&1".format(self.java_home, src_dir, command, log_file)
+        build_cmd = "export JAVA_HOME={}; cd {}; {} < /dev/null > {} 2>&1".format(self.java_home, src_dir, command, log_file)
         self.logger.info("Running build command [%s]", build_cmd)
 
         if process.run_subprocess(build_cmd):
diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/config.ini b/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/config.ini
index 1fdb47ad..a09e14e2 100644
--- a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/config.ini
+++ b/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/config.ini
@@ -16,7 +16,7 @@ jdk.unbundled.release_url = https://artifacts.opensearch.org/releases/bundle/ope
 
 docker_image=opensearchproject/opensearch
 # major version of the JDK that is used to build OpenSearch
-build.jdk = 17
+build.jdk = 21
 # list of JDK major versions that are used to run OpenSearch
-runtime.jdk = 17,16,15,14,13,12,11,8
+runtime.jdk = 21,17,16,15,14,13,12,11,8
 runtime.jdk.bundled = true
diff --git a/osbenchmark/utils/process.py b/osbenchmark/utils/process.py
index 4cc54027..65dcbc92 100644
--- a/osbenchmark/utils/process.py
+++ b/osbenchmark/utils/process.py
@@ -40,7 +40,7 @@ def run_subprocess_with_output(command_line):
     logger.debug("Running subprocess [%s] with output.", command_line)
     command_line_args = shlex.split(command_line)
 
-    with subprocess.Popen(command_line_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as command_line_process:
+    with subprocess.Popen(command_line_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL) as command_line_process:
         has_output = True
         lines = []
         while has_output:
@@ -57,7 +57,7 @@ def run_subprocess_with_out_and_err(command_line):
     logger.debug("Running subprocess [%s] with stdout and stderr.", command_line)
     command_line_args = shlex.split(command_line)
 
-    sp = subprocess.Popen(command_line_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    sp = subprocess.Popen(command_line_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL)
     sp.wait()
     out, err = sp.communicate()
     return out.decode('UTF-8'), err.decode('UTF-8'), sp.returncode
@@ -68,7 +68,7 @@ def run_subprocess_with_stderr(command_line):
     logger.debug("Running subprocess [%s] with stderr but no stdout.", command_line)
     command_line_args = shlex.split(command_line)
 
-    sp = subprocess.Popen(command_line_args, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+    sp = subprocess.Popen(command_line_args, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL)
     sp.wait()
     _, err = sp.communicate()
     return err.decode('UTF-8'), sp.returncode
@@ -91,7 +91,7 @@ def exit_status_as_bool(runnable, quiet=False):
 
 
 def run_subprocess_with_logging(command_line, header=None, level=logging.INFO, stdin=None, stdout=subprocess.PIPE,
-                                stderr=subprocess.STDOUT, env=None, detach=False):
+                                stderr=subprocess.STDOUT, env=None, detach=False, capture_output=False):
     """
     Runs the provided command line in a subprocess. All output will be captured by a logger.
 
@@ -128,7 +128,7 @@ def run_subprocess_with_logging(command_line, header=None, level=logging.INFO, s
             logger.log(level=level, msg=stdout)
 
     logger.debug("Subprocess [%s] finished with return code [%s].", command_line, str(command_line_process.returncode))
-    return command_line_process.returncode
+    return (stdout, command_line_process.returncode) if capture_output else command_line_process.returncode
 
 
 def is_benchmark_process(p):
diff --git a/tests/builder/provisioner_test.py b/tests/builder/provisioner_test.py
index 4b1539f5..a6ffbe10 100644
--- a/tests/builder/provisioner_test.py
+++ b/tests/builder/provisioner_test.py
@@ -243,7 +243,7 @@ def test_invokes_hook_no_java_home(self):
 class PluginInstallerTests(TestCase):
     @mock.patch("osbenchmark.utils.process.run_subprocess_with_logging")
     def test_install_plugin_successfully(self, installer_subprocess):
-        installer_subprocess.return_value = 0
+        installer_subprocess.return_value = "output", 0
 
         plugin = provision_config.PluginDescriptor(name="unit-test-plugin", config="default", variables={"active": True})
         installer = provisioner.PluginInstaller(plugin,
@@ -254,11 +254,11 @@ def test_install_plugin_successfully(self, installer_subprocess):
 
         installer_subprocess.assert_called_with(
             '/opt/opensearch/bin/opensearch-plugin install --batch "unit-test-plugin"',
-            env={"JAVA_HOME": "/usr/local/javas/java8"})
+            env={"JAVA_HOME": "/usr/local/javas/java8"}, capture_output=True)
 
     @mock.patch("osbenchmark.utils.process.run_subprocess_with_logging")
     def test_install_plugin_with_bundled_jdk(self, installer_subprocess):
-        installer_subprocess.return_value = 0
+        installer_subprocess.return_value = "output", 0
 
         plugin = provision_config.PluginDescriptor(name="unit-test-plugin", config="default", variables={"active": True})
         installer = provisioner.PluginInstaller(plugin,
@@ -270,12 +270,12 @@ def test_install_plugin_with_bundled_jdk(self, installer_subprocess):
 
         installer_subprocess.assert_called_with(
             '/opt/opensearch/bin/opensearch-plugin install --batch "unit-test-plugin"',
-            env={})
+            env={}, capture_output=True)
 
     @mock.patch("osbenchmark.utils.process.run_subprocess_with_logging")
     def test_install_unknown_plugin(self, installer_subprocess):
         # unknown plugin
-        installer_subprocess.return_value = 64
+        installer_subprocess.return_value = "output", 64
 
         plugin = provision_config.PluginDescriptor(name="unknown")
         installer = provisioner.PluginInstaller(plugin,
@@ -288,12 +288,12 @@ def test_install_unknown_plugin(self, installer_subprocess):
 
         installer_subprocess.assert_called_with(
             '/opt/opensearch/bin/opensearch-plugin install --batch "unknown"',
-            env={"JAVA_HOME": "/usr/local/javas/java8"})
+            env={"JAVA_HOME": "/usr/local/javas/java8"}, capture_output=True)
 
     @mock.patch("osbenchmark.utils.process.run_subprocess_with_logging")
     def test_install_plugin_with_io_error(self, installer_subprocess):
         # I/O error
-        installer_subprocess.return_value = 74
+        installer_subprocess.return_value = "output", 74
 
         plugin = provision_config.PluginDescriptor(name="simple")
         installer = provisioner.PluginInstaller(plugin,
@@ -306,12 +306,12 @@ def test_install_plugin_with_io_error(self, installer_subprocess):
 
         installer_subprocess.assert_called_with(
             '/opt/opensearch/bin/opensearch-plugin install --batch "simple"',
-            env={"JAVA_HOME": "/usr/local/javas/java8"})
+            env={"JAVA_HOME": "/usr/local/javas/java8"}, capture_output=True)
 
     @mock.patch("osbenchmark.utils.process.run_subprocess_with_logging")
     def test_install_plugin_with_unknown_error(self, installer_subprocess):
         # some other error
-        installer_subprocess.return_value = 12987
+        installer_subprocess.return_value = "output", 12987
 
         plugin = provision_config.PluginDescriptor(name="simple")
         installer = provisioner.PluginInstaller(plugin,
@@ -320,12 +320,12 @@ def test_install_plugin_with_unknown_error(self, installer_subprocess):
 
         with self.assertRaises(exceptions.BenchmarkError) as ctx:
             installer.install(os_home_path="/opt/opensearch")
-        self.assertEqual("Unknown error while trying to install [simple] (installer return code [12987]). Please check the logs.",
+        self.assertEqual("Unknown error 'output' while trying to install [simple] (installer return code [12987]). Please check the logs.",
                          ctx.exception.args[0])
 
         installer_subprocess.assert_called_with(
             '/opt/opensearch/bin/opensearch-plugin install --batch "simple"',
-            env={"JAVA_HOME": "/usr/local/javas/java8"})
+            env={"JAVA_HOME": "/usr/local/javas/java8"}, capture_output=True)
 
     def test_pass_plugin_properties(self):
         plugin = provision_config.PluginDescriptor(name="unit-test-plugin",
diff --git a/tests/builder/supplier_test.py b/tests/builder/supplier_test.py
index 25944705..75b99d05 100644
--- a/tests/builder/supplier_test.py
+++ b/tests/builder/supplier_test.py
@@ -152,9 +152,9 @@ def test_build_on_jdk_8(self, jvm_resolve_path, mock_run_subprocess):
 
         calls = [
             # Actual call
-            mock.call("export JAVA_HOME=/opt/jdk8; cd /src; ./gradlew clean > logs/build.log 2>&1"),
+            mock.call("export JAVA_HOME=/opt/jdk8; cd /src; ./gradlew clean < /dev/null > logs/build.log 2>&1"),
             # Return value check
-            mock.call("export JAVA_HOME=/opt/jdk8; cd /src; ./gradlew assemble > logs/build.log 2>&1"),
+            mock.call("export JAVA_HOME=/opt/jdk8; cd /src; ./gradlew assemble < /dev/null > logs/build.log 2>&1"),
         ]
 
         mock_run_subprocess.assert_has_calls(calls)
@@ -170,9 +170,9 @@ def test_build_on_jdk_10(self, jvm_resolve_path, mock_run_subprocess):
 
         calls = [
             # Actual call
-            mock.call("export JAVA_HOME=/opt/jdk10; cd /src; ./gradlew clean > logs/build.log 2>&1"),
+            mock.call("export JAVA_HOME=/opt/jdk10; cd /src; ./gradlew clean < /dev/null > logs/build.log 2>&1"),
             # Return value check
-            mock.call("export JAVA_HOME=/opt/jdk10; cd /src; ./gradlew assemble > logs/build.log 2>&1"),
+            mock.call("export JAVA_HOME=/opt/jdk10; cd /src; ./gradlew assemble < /dev/null > logs/build.log 2>&1"),
         ]
 
         mock_run_subprocess.assert_has_calls(calls)
diff --git a/tox.ini b/tox.ini
index e1283eee..92a98a1f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -16,16 +16,7 @@ deps=
     pytest
 passenv =
     HOME
-    JAVA_HOME
-    JAVA8_HOME
-    JAVA9_HOME
-    JAVA10_HOME
-    JAVA11_HOME
-    JAVA12_HOME
-    JAVA13_HOME
-    JAVA14_HOME
-    JAVA15_HOME
-    JAVA16_HOME
+    JAVA*_HOME
     BENCHMARK_HOME
     SSH_AUTH_SOCK
     THESPLOG_FILE