Implement searching password hashes using split database

Documentation/PowerShell/Test-PasswordQuality.md

@@ -12,12 +12,20 @@ Performs AD audit, including checks for weak, duplicate, default and empty passw
 
 ## SYNTAX
 
+### Using monolithic password hashes sorted file (HIBP v8 and earlier)
 ```
 Test-PasswordQuality [-Account] <DSAccount> [-SkipDuplicatePasswordTest] [-IncludeDisabledAccounts]
  [-WeakPasswords <String[]>] [-WeakPasswordsFile <String>] [-WeakPasswordHashesFile <String>]
  [-WeakPasswordHashesSortedFile <String>] [<CommonParameters>]
 ```
 
+### Using multiple password hashes sorted files (HIBP after v8)
+```
+Test-PasswordQuality [-Account] <DSAccount> [-SkipDuplicatePasswordTest] [-IncludeDisabledAccounts]
+ [-WeakPasswords <String[]>] [-WeakPasswordsFile <String>] [-WeakPasswordHashesFile <String>]
+ [-WeakPasswordHashesSortedFilePath <String>] [<CommonParameters>]
+```
+
 ## DESCRIPTION
 
 The Test-PasswordQuality cmdlet is a simple tool for Active Directory password auditing. It can detect weak, duplicate, default, non-expiring or empty passwords and find accounts that are violating security best practices. The cmdlet accepts output of the Get-ADDBAccount and Get-ADReplAccount cmdlets, so both offline (ntds.dit) and online (DCSync) password analysis can be done.
@@ -31,7 +39,7 @@ Although the cmdlet output is formatted in a human readable fashion, it is still
 ### Example 1
 ```powershell
 PS C:\> Get-ADDBAccount -All -DatabasePath ntds.dit -BootKey acdba64a3929261b04e5270c3ef973cf |
-            Test-PasswordQuality -WeakPasswordHashesSortedFile pwned-passwords-ntlm-ordered-by-hash-v5.txt
+            Test-PasswordQuality -WeakPasswordHashesSortedFilePath P:\pwnedpasswords_ntlm
 <# Sample Output:
 
 Active Directory Password Quality Report
@@ -102,7 +110,7 @@ Performs an offline credential hygiene audit of AD database against HIBP.
 ```powershell
 PS C:\> $results = Get-ADReplAccount -All -Server LON-DC1 |
                    Test-PasswordQuality -WeakPasswords 'Pa$$w0rd','April2019' `
-                                        -WeakPasswordHashesSortedFile pwned-passwords-ntlm-ordered-by-hash-v5.txt
+                                        -WeakPasswordHashesSortedFile pwned-passwords-ntlm-ordered-by-hash-v8.txt
 ```
 
 Performs an online credential hygiene audit of AD against HIBP + a custom wordlist.
@@ -120,7 +128,7 @@ Performs a dictionary attack against a set of accounts. The Test-PasswordQuality
 ```powershell
 PS C:\> Get-ADDBAccount -All -DatabasePath ntds.dit -BootKey $key |
             where DistinguishedName -like '*OU=Employees,DC=contoso,DC=com' |
-            Test-PasswordQuality -IncludeDisabledAccounts -WeakPasswordHashesSortedFile pwned-passwords-ntlm-ordered-by-hash-v5.txt
+            Test-PasswordQuality -IncludeDisabledAccounts -WeakPasswordHashesSortedFilePath P:\pwnedpasswords_ntlm
 ```
 
 Performs an offline credential hygiene audit of a selected OU from AD database against HIBP.
@@ -209,11 +217,26 @@ Accept wildcard characters: False
 ```
 
 ### -WeakPasswordHashesSortedFile
-Path to a file that contains NT hashes of weak passwords, one hash in HEX format per line. The hashes must be sorted alphabetically, because a binary search is performed. This parameter is typically used with a list of leaked password hashes from HaveIBeenPwned.
+Path to a file that contains NT hashes of weak passwords, one hash in HEX format per line. The hashes must be sorted alphabetically, because a binary search is performed. This parameter is typically used with a list of leaked password hashes from HaveIBeenPwned, v8 and earlier.
 
 ```yaml
 Type: String
-Parameter Sets: (All)
+Parameter Sets: SingleFile
+Aliases:
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
+### -WeakPasswordHashesSortedFilePath
+Path to a directory of files named as the first five characters of an NT hash (00000.txt - FFFFF.txt), each of which contain NT hashes of weak passwords, one hash in HEX format per line, starting with the 6th character in the hash. The hashes must be sorted alphabetically, because a binary search is performed. This parameter is typically used with a list of leaked password hashes from HaveIBeenPwned after v8.
+
+```yaml
+Type: String
+Parameter Sets: MultiFile
 Aliases:
 
 Required: False

Src/DSInternals.PowerShell/Commands/Misc/TestPasswordQualityCommand.cs

@@ -15,6 +15,10 @@
     public class TestPasswordQualityCommand : PSCmdletEx, IDisposable
     {
         #region Constants
+        protected const string ParamSetSingleSortedFile = "SingleFile";
+
+        protected const string ParamSetMultipuleSortedFile = "MultiFile";
+
         /// <summary>
         /// Expected number of users being processed.
         /// </summary>
@@ -87,13 +91,21 @@ public string WeakPasswordHashesFile
             set;
         }
 
-        [Parameter]
+        [Parameter(ParameterSetName = ParamSetSingleSortedFile)]
         [ValidateNotNullOrEmpty]
         public string WeakPasswordHashesSortedFile
         {
             get;
             set;
         }
+
+        [Parameter(ParameterSetName = ParamSetMultipuleSortedFile)]
+        [ValidateNotNullOrEmpty]
+        public string WeakPasswordHashesSortedFilePath
+        {
+            get;
+            set;
+        }
         #endregion Parameters
 
         #region Fields
@@ -115,10 +127,11 @@ protected override void BeginProcessing()
             // Test the optional file path in advance to throw an early error.
             this.ResolveFilePath(this.WeakPasswordHashesFile);
             this.ResolveFilePath(this.WeakPasswordsFile);
+            this.ResolveDirectoryPath(this.WeakPasswordHashesSortedFilePath);
 
             // Open the sorted weak password hashes file, as we will be searching it on-the-fly.
             string sortedHashesFile = this.ResolveFilePath(this.WeakPasswordHashesSortedFile);
-            if(sortedHashesFile != null)
+            if (sortedHashesFile != null)
             {
                 this.sortedHashFileSearcher = new SortedFileSearcher(sortedHashesFile);
             }
@@ -294,10 +307,26 @@ private bool ShouldTestWeakPasswordsInMemory
 
         private void LookupAccountNTHashInSortedFile()
         {
+            string hash = this.Account.NTHash.ToHex(true);
+            // If there is a file path present, the hashes are in seperate sorted files
+            if (this.WeakPasswordHashesSortedFilePath != null)
+            {
+                // The files in the path should be named with the first 5 chararacters of the hash and the extension txt, like ABDD0.txt
+                string sortedHashesFile = this.ResolveFilePath(this.WeakPasswordHashesSortedFilePath + hash.Substring(0, 5) + ".txt");
+                if (sortedHashesFile != null)
+                {
+                    // Assuming all went well, we should be able to set up to search this much smaller file for the hashes
+                    this.sortedHashFileSearcher = new SortedFileSearcher(sortedHashesFile);
+
+                    // In the split database the hashes are stored in the sorted files starting with the 6th character (since the filename is the first 5
+                    hash = hash.Substring(5);
+                }
+            }
+
             if (this.sortedHashFileSearcher != null)
             {
                 // Check the password on the fly in the sorted file using binary search
-                bool found = this.sortedHashFileSearcher.FindString(this.Account.NTHash.ToHex());
+                bool found = this.sortedHashFileSearcher.FindString(hash);
                 if (found)
                 {
                     this.result.WeakPassword.UnionWith(new string[] { this.Account.LogonName });

Src/DSInternals.PowerShell/en-US/DSInternals.PowerShell.dll-Help.xml

@@ -3606,7 +3606,7 @@ Credential Roaming
         <dev:code>PS C:\&gt; $results = Get-ADDBAccount -DatabasePath '.\Active Directory\ntds.dit' `
                                    -BootKey acdba64a3929261b04e5270c3ef973cf `
                                    -All  |
-                   Test-PasswordQuality -WeakPasswordHashesSortedFile pwned-passwords-ntlm-ordered-by-hash-v5.txt</dev:code>
+                   Test-PasswordQuality -WeakPasswordHashesSortedFile pwned-passwords-ntlm-ordered-by-hash-v8.txt</dev:code>
         <dev:remarks>
           <maml:para>Performs an offline credential hygiene audit of AD database against HIBP.</maml:para>
         </dev:remarks>
@@ -5572,7 +5572,7 @@ Credential Roaming
       <command:example>
         <maml:title>-------------------------- Example 3 --------------------------</maml:title>
         <dev:code>PS C:\&gt; $results = Get-ADReplAccount -All -Server 'lon-dc1.contoso.com' |
-                   Test-PasswordQuality -WeakPasswordHashesSortedFile pwned-passwords-ntlm-ordered-by-hash-v5.txt</dev:code>
+                   Test-PasswordQuality -WeakPasswordHashesSortedFile pwned-passwords-ntlm-ordered-by-hash-v8.txt</dev:code>
         <dev:remarks>
           <maml:para>Performs an online credential hygiene audit of AD against HIBP.</maml:para>
         </dev:remarks>
@@ -10947,7 +10947,19 @@ PS C:\&gt; Set-AzureADUserEx -UserPrincipalName '[email protected]' -KeyCredentia
         <command:parameter required="false" variableLength="true" globbing="false" pipelineInput="False" position="named" aliases="none">
           <maml:name>WeakPasswordHashesSortedFile</maml:name>
           <maml:Description>
-            <maml:para>Path to a file that contains NT hashes of weak passwords, one hash in HEX format per line. The hashes must be sorted alphabetically, because a binary search is performed. This parameter is typically used with a list of leaked password hashes from HaveIBeenPwned.</maml:para>
+            <maml:para>Path to a file that contains NT hashes of weak passwords, one hash in HEX format per line. The hashes must be sorted alphabetically, because a binary search is performed. This parameter is typically used with a list of leaked password hashes from HaveIBeenPwned, v8 and earlier.</maml:para>
+          </maml:Description>
+          <command:parameterValue required="true" variableLength="false">String</command:parameterValue>
+          <dev:type>
+            <maml:name>String</maml:name>
+            <maml:uri />
+          </dev:type>
+          <dev:defaultValue>None</dev:defaultValue>
+        </command:parameter>
+        <command:parameter required="false" variableLength="true" globbing="false" pipelineInput="False" position="named" aliases="none">
+          <maml:name>WeakPasswordHashesSortedFilePath</maml:name>
+          <maml:Description>
+            <maml:para>Path to a directory of files named as the first five characters of an NT hash (00000.txt - FFFFF.txt), each of which contain NT hashes of weak passwords, one hash in HEX format per line, starting with the 6th character in the hash. The hashes must be sorted alphabetically, because a binary search is performed. This parameter is typically used with a list of leaked password hashes from HaveIBeenPwned after v8.</maml:para>
           </maml:Description>
           <command:parameterValue required="true" variableLength="false">String</command:parameterValue>
           <dev:type>
@@ -11097,8 +11109,8 @@ PS C:\&gt; Set-AzureADUserEx -UserPrincipalName '[email protected]' -KeyCredentia
       <command:example>
         <maml:title>-------------------------- Example 1 --------------------------</maml:title>
         <dev:code>PS C:\&gt; Get-ADDBAccount -All -DatabasePath ntds.dit -BootKey acdba64a3929261b04e5270c3ef973cf |
-            Test-PasswordQuality -WeakPasswordHashesSortedFile pwned-passwords-ntlm-ordered-by-hash-v5.txt
-&lt;# Sample Output:
+          Test-PasswordQuality -WeakPasswordHashesSortedFilePath P:\pwnedpasswords_ntlm
+          &lt;# Sample Output:
 
 Active Directory Password Quality Report
 ----------------------------------------
@@ -11168,7 +11180,7 @@ These accounts that require smart card authentication have a password:
         <maml:title>-------------------------- Example 2 --------------------------</maml:title>
         <dev:code>PS C:\&gt; $results = Get-ADReplAccount -All -Server LON-DC1 |
                    Test-PasswordQuality -WeakPasswords 'Pa$$w0rd','April2019' `
-                                        -WeakPasswordHashesSortedFile pwned-passwords-ntlm-ordered-by-hash-v5.txt</dev:code>
+                                        -WeakPasswordHashesSortedFile pwned-passwords-ntlm-ordered-by-hash-v8.txt</dev:code>
         <dev:remarks>
           <maml:para>Performs an online credential hygiene audit of AD against HIBP + a custom wordlist.</maml:para>
         </dev:remarks>
@@ -11653,4 +11665,4 @@ These groups of accounts have the same passwords:
       </maml:navigationLink>
     </command:relatedLinks>
   </command:command>
-</helpItems>
+</helpItems>