diff --git a/.github/actions/run-released-opensearch/action.yml b/.github/actions/run-released-opensearch/action.yml
index 53c924ce..c77356f0 100644
--- a/.github/actions/run-released-opensearch/action.yml
+++ b/.github/actions/run-released-opensearch/action.yml
@@ -11,6 +11,9 @@ outputs:
   opensearch_url:
     description: The URL where the OpenSearch node is accessible
     value: ${{ steps.opensearch.outputs.url }}
+  admin_password:
+    description: The initial admin password
+    value: ${{ steps.opensearch.outputs.admin_password }}
 runs:
   using: composite
   steps:
diff --git a/.github/actions/start-opensearch/action.yml b/.github/actions/start-opensearch/action.yml
index 306a9079..d23e720c 100644
--- a/.github/actions/start-opensearch/action.yml
+++ b/.github/actions/start-opensearch/action.yml
@@ -8,55 +8,66 @@ outputs:
   url:
     description: The URL where the OpenSearch node is accessible
     value: ${{ steps.opensearch.outputs.url }}
+  admin_password:
+    description: The initial admin password
+    value: ${{ steps.opensearch.outputs.password }}
 runs:
   using: composite
   steps:
-    - name: Install Java
-      uses: actions/setup-java@v3
-      with:
-        distribution: zulu
-        java-version: 11
-
     - name: Start OpenSearch
       id: opensearch
-      shell: bash -eo pipefail {0}
+      shell: bash -exo pipefail {0}
       run: |
         if [[ "$RUNNER_OS" == "macOS" ]]; then
           brew install -q coreutils
         fi
-        OPENSEARCH_HOME=$(realpath ./opensearch-*)
+        OPENSEARCH_HOME=$(realpath ./opensearch-[1-9]*)
+        CONFIG_DIR=$OPENSEARCH_HOME/config
+        CONFIG_FILE=$CONFIG_DIR/opensearch.yml
+        SECURITY_DIR=$OPENSEARCH_HOME/plugins/opensearch-security
         OPENSEARCH_JAVA_OPTS="-Djava.net.preferIPv4Stack=true"
-        export OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin
 
-        url="http://localhost:9200"
-        cp ./client/.ci/opensearch/opensearch.yml $OPENSEARCH_HOME/config/
+        URL="http://localhost:9200"
+        cp ./client/.ci/opensearch/opensearch.yml $CONFIG_FILE
 
         bash ./client/.ci/generate-certs.sh
+        
+        export OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin
 
-        if [[ -d "$OPENSEARCH_HOME/plugins/opensearch-security" ]]; then
+        if [[ -d "$SECURITY_DIR" ]]; then
           if [[ "$SECURED" == "true" ]]; then
-            OPENSEARCH_VERSION=$(cat $OPENSEARCH_HOME/plugins/opensearch-security/plugin-descriptor.properties | grep '^version=' | cut -d'=' -f 2)
-            OPENSEARCH_REQUIRED_VERSION="2.12.0"
-            # Starting in 2.12.0, security demo configuration script requires an initial admin password
-            COMPARE_VERSION=`echo $OPENSEARCH_REQUIRED_VERSION $OPENSEARCH_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1`
-            if [ "$COMPARE_VERSION" == "$OPENSEARCH_REQUIRED_VERSION" ]; then
-              bash $OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s -t
+            SECURITY_VERSION=$(cat $SECURITY_DIR/plugin-descriptor.properties | grep '^version=' | cut -d'=' -f 2)
+            SECURITY_VERSION_COMPONENTS=(${SECURITY_VERSION//./ })
+            SECURITY_MAJOR="${SECURITY_VERSION_COMPONENTS[0]}"
+            SECURITY_MINOR="${SECURITY_VERSION_COMPONENTS[1]}"
+        
+            if (( $SECURITY_MAJOR > 2 || ( $SECURITY_MAJOR == 2 && $SECURITY_MINOR >= 12 ) )); then
+              export OPENSEARCH_INITIAL_ADMIN_PASSWORD="myStrongPassword123!"
+            fi
+        
+            if [[ "$RUNNER_OS" != "Windows" ]]; then
+              bash "$SECURITY_DIR/tools/install_demo_configuration.sh" -y -i -s
             else
-              bash $OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s
+              "$SECURITY_DIR/tools/install_demo_configuration.bat" -y -i -s
             fi
-            sed -i.bak -e 's/plugins.security.audit.type:.*/plugins.security.audit.type: log4j/' $OPENSEARCH_HOME/config/opensearch.yml
-            cp ./client/.ci/opensearch/*.pem $OPENSEARCH_HOME/config/
-            url="https://localhost:9200"
+        
+            sed -i.bak -e 's/plugins.security.audit.type:.*/plugins.security.audit.type: log4j/' $CONFIG_FILE
+            cp ./client/.ci/opensearch/*.pem $CONFIG_DIR/
+            URL="https://localhost:9200"
           else
-            printf "\nplugins.security.disabled: true" >> $OPENSEARCH_HOME/config/opensearch.yml
+            printf "\nplugins.security.disabled: true" >> $CONFIG_FILE
           fi
         fi
 
         if [[ "$RUNNER_OS" == "macOS" ]]; then
-          sed -i.bak -e 's/bootstrap.memory_lock:.*/bootstrap.memory_lock: false/' $OPENSEARCH_HOME/config/opensearch.yml
+          sed -i.bak -e 's/bootstrap.memory_lock:.*/bootstrap.memory_lock: false/' $CONFIG_FILE
         fi
 
-        echo "url=$url" >> $GITHUB_OUTPUT
+        {
+          echo "url=$URL"
+          echo "password=$OPENSEARCH_INITIAL_ADMIN_PASSWORD"
+        } | tee -a $GITHUB_OUTPUT
+        
         if [[ "$RUNNER_OS" == "Linux" ]]; then
           sudo swapoff -a
           sudo sysctl -w vm.swappiness=1
@@ -73,7 +84,7 @@ runs:
 
         for attempt in {1..20}; do
           sleep 5
-          if curl -k -sS --cacert ./client/.ci/certs/root-ca.crt -u admin:admin $url; then
+          if curl -k -sS --cacert ./client/.ci/certs/root-ca.crt -u admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD} $URL; then
             echo '=====> ready'
             exit 0
           fi
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7746f37c..c748004a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -29,11 +29,18 @@ jobs:
       - name: Setup Rust tools
         uses: ./client/.github/actions/setup-rust-tools
 
+      - name: Install Java
+        if: matrix.os == 'windows-latest'
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+
       - name: Run OpenSearch
         id: opensearch
         uses: ./client/.github/actions/run-released-opensearch
         with:
-          version: 2.8.0
+          version: 2.16.0
           secured: true
 
       - name: Run Tests
@@ -41,6 +48,7 @@ jobs:
         run: cargo make test ${{ matrix.test-args }}
         env:
           OPENSEARCH_URL: ${{ steps.opensearch.outputs.opensearch_url }}
+          OPENSEARCH_PASSWORD: ${{ steps.opensearch.outputs.admin_password }}
 
       - name: Upload Coverage Data
         uses: codecov/codecov-action@v4
@@ -63,7 +71,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: ['1.1.0', '1.2.4', '1.3.4', '2.2.0', '2.4.0', '2.6.0', '2.8.0', '2.12.0']
+        version:
+          - 2.16.0
+          - 2.14.0
+          - 2.12.0
+          - 2.10.0
+          - 2.8.0
+          - 2.6.0
+          - 2.4.1
+          - 2.2.1
+          - 2.0.1
+          - 1.3.14
+          - 1.2.4
+          - 1.1.0
         secured: [true, false]
     steps:
       - name: Checkout Rust Client
@@ -86,6 +106,7 @@ jobs:
         run: cargo make test-yaml
         env:
           OPENSEARCH_URL: ${{ steps.opensearch.outputs.opensearch_url }}
+          OPENSEARCH_PASSWORD: ${{ steps.opensearch.outputs.admin_password }}
 
       - uses: codecov/codecov-action@v4
         with:
@@ -106,7 +127,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        opensearch_ref: ['1.x', '2.0', '2.x', 'main']
+        include:
+          - { opensearch_ref: '1.x', java_version: '11' }
+          - { opensearch_ref: '2.x', java_version: '17' }
+          - { opensearch_ref: 'main', java_version: '21' }
     steps:
       - name: GitHub App token
         id: github_app_token
@@ -145,6 +169,12 @@ jobs:
           path: opensearch-*
           key: opensearch-${{ steps.opensearch-git.outputs.sha }}
 
+      - name: Install Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java_version }}
+
       - name: Build OpenSearch
         if: steps.cache-restore.outputs.cache-hit != 'true'
         shell: bash -eo pipefail {0}
@@ -170,6 +200,7 @@ jobs:
         run: cargo make test-yaml
         env:
           OPENSEARCH_URL: ${{ steps.opensearch.outputs.url }}
+          OPENSEARCH_PASSWORD: ${{ steps.opensearch.outputs.admin_password }}
 
       - uses: codecov/codecov-action@v4
         with:
diff --git a/opensearch/examples/cat_indices.rs b/opensearch/examples/cat_indices.rs
index 1fb8335e..e7f58c37 100644
--- a/opensearch/examples/cat_indices.rs
+++ b/opensearch/examples/cat_indices.rs
@@ -70,7 +70,10 @@ fn create_client() -> Result<OpenSearch, Error> {
         let system = System::new_with_specifics(
             RefreshKind::new().with_processes(ProcessRefreshKind::default()),
         );
-        let has_fiddler = system.processes_by_name(OsStr::new("Fiddler")).next().is_some();
+        let has_fiddler = system
+            .processes_by_name(OsStr::new("Fiddler"))
+            .next()
+            .is_some();
         has_fiddler
     }
 
diff --git a/opensearch/tests/common/client.rs b/opensearch/tests/common/client.rs
index f1c56a1c..974c46c7 100644
--- a/opensearch/tests/common/client.rs
+++ b/opensearch/tests/common/client.rs
@@ -84,7 +84,10 @@ impl TestClientBuilder {
         // assume if we're running with HTTPS then authentication is also enabled and disable
         // certificate validation - we'll change this for tests that need to.
         if secure {
-            builder = builder.auth(Credentials::Basic("admin".into(), "admin".into()));
+            builder = builder.auth(Credentials::Basic(
+                "admin".into(),
+                std::env::var("OPENSEARCH_PASSWORD").unwrap_or("admin".into()),
+            ));
 
             #[cfg(any(feature = "native-tls", feature = "rustls-tls"))]
             {
diff --git a/yaml_test_runner/src/main.rs b/yaml_test_runner/src/main.rs
index 840ba15f..e9b876e6 100644
--- a/yaml_test_runner/src/main.rs
+++ b/yaml_test_runner/src/main.rs
@@ -156,7 +156,13 @@ fn branch_suite_and_version_from_opensearch(
 
     let suite = TestSuite::Free;
 
-    let response = client.get(url).basic_auth("admin", Some("admin")).send()?;
+    let response = client
+        .get(url)
+        .basic_auth(
+            "admin",
+            Some(std::env::var("OPENSEARCH_PASSWORD").unwrap_or("admin".into())),
+        )
+        .send()?;
     let json: Value = response.json()?;
     let branch = match json["version"]["build_hash"].as_str() {
         Some(build_hash) if build_hash != "unknown" => build_hash.to_string(),
diff --git a/yaml_test_runner/tests/common/client.rs b/yaml_test_runner/tests/common/client.rs
index d01f4590..96412c72 100644
--- a/yaml_test_runner/tests/common/client.rs
+++ b/yaml_test_runner/tests/common/client.rs
@@ -62,7 +62,10 @@ fn cluster_addr() -> String {
 fn running_proxy() -> bool {
     let mut system = sysinfo::System::new();
     system.refresh_processes(sysinfo::ProcessesToUpdate::All);
-    let running = system.processes_by_name(OsStr::new("Fiddler")).next().is_some();
+    let running = system
+        .processes_by_name(OsStr::new("Fiddler"))
+        .next()
+        .is_some();
     running
 }