Update User Audit Scripts for Pagination

User Audit/README.md

@@ -33,5 +33,8 @@ You can set an environment variable in Windows using the setx command in the com
 In macOS or Linux, you can set an environment variable in the terminal using the export command:
 *export API_KEY=your-api-key*
 
+### 4. Pagination Size
+List Users endpoint is paginated. Pagination size is set to 500 in both the scripts. If you want a higher or lower pagination size please change `per_page` variable's value in the script.
+
 ### 3: Fetch Audit Logs 
 Update the start_date (Line #70) and end_date (Line #71) in the script as per the time period you want to pull the logs for. Based on this time period, the script will check which user used their API keys in that time frame.

User Audit/audit_test_full.py

@@ -32,15 +32,28 @@ def flatten(x, name='', exclude=exclude):
 
 token = os.environ.get('API_KEY')
 base_url = "https://api.kennasecurity.com"
-users_url= base_url + "/users"
+per_page = 500
+users_url= base_url + "/users?per_page=" + str(per_page)
 roles_url = base_url + "/roles"
 audit_logs_url = base_url + "/audit_logs/"
 
 headers = {"Accept": "application/json", "X-Risk-Token":token}
+pages = -1
+page = 1
+users_df = pd.DataFrame()
+
+while True:
+    paged_url = users_url + "&page=" + str(page)
+    print("Requesting data from", paged_url)
+    users_response = requests.get(paged_url, headers=headers).json()
+    users_df = pd.concat([users_df, pd.DataFrame(json_normalize([flatten_json(x) for x in users_response['users']]))], ignore_index=True)   
+    # do this once
+    if 'meta' in users_response and pages == -1:
+        pages = users_response['meta']['pages']
+    if page >= pages:
+        break
+    page += 1
 
-users_response = requests.get(users_url, headers=headers).json()
-
-users_df = pd.DataFrame(json_normalize([flatten_json(x) for x in users_response['users']]))
 users_df = users_df.rename(columns={"id":"user_id","created_at":"user_created_at","updated_at":"user_updated_at"})
 
 users_df['user_created_at'] = pd.to_datetime(users_df['user_created_at'], format='%Y-%m-%d', errors='coerce').dt.date
@@ -126,36 +139,33 @@ def flatten(x, name='', exclude=exclude):
     audit_logs_processed.append(event_data)
 
 audit_df = pd.DataFrame(audit_logs_processed)
-
-# Open the existing workbook with 'openpyxl'
 wb = load_workbook('cvm_user_audit.xlsx')
+users_sheet = wb['Users']
 
-# Write the 'Audit Logs' DataFrame to the workbook
-with pd.ExcelWriter('cvm_user_audit.xlsx', engine='openpyxl') as writer:
-    writer.book = wb
-    merged_df = pd.merge(users_df, audit_df, left_on="user_id", right_on="kenna_user_id", how="inner")
-    merged_df.to_excel(writer, sheet_name='Audit Logs')
+# Define a fill for highlighting cells
+green_fill = PatternFill(start_color='00FF00', end_color='00FF00', fill_type='solid')
 
-    # Get the 'Users' sheet
-    users_sheet = wb['Users']
+# Get emails from audit logs data with 'source' as 'API'
+api_emails = [log['user_email'] for log in audit_logs_processed if log['source'] == 'API']
 
-    # Define a fill for highlighting cells
-    green_fill = PatternFill(start_color='00FF00', end_color='00FF00', fill_type='solid')
+# Iterate over the rows in the 'Users' DataFrame
+for i, row in users_df.iterrows():
+    email = row['email']  # Assuming 'email' is a column in your DataFrame
+    # Check if the email is in api_emails
+    if email in api_emails:
+        # If it is, highlight the entire row
+        for j in range(1, len(row) + 1):
+            print("Highlighting")
+            users_sheet.cell(row=i+2, column=j).fill = green_fill  # i+2 because DataFrame is 0-indexed and Worksheet is 1-indexed, and we have a header row
 
-    # Get emails from audit logs data with 'source' as 'API'
-    api_emails = [log['user_email'] for log in audit_logs_processed if log['source'] == 'API']
+# Save the workbook
+wb.save('cvm_user_audit.xlsx')
 
-    # Iterate over the rows in the 'Users' DataFrame
-    for i, row in users_df.iterrows():
-        email = row['email']  # Assuming 'email' is a column in your DataFrame
-        # Check if the email is in api_emails
-        if email in api_emails:
-            # If it is, highlight the entire row
-            for j in range(1, len(row) + 1):
-                users_sheet.cell(row=i+2, column=j).fill = green_fill  # i+2 because DataFrame is 0-indexed and Worksheet is 1-indexed, and we have a header row
+# Write the 'Audit Logs' DataFrame to the workbook
+with pd.ExcelWriter('cvm_user_audit.xlsx', engine='openpyxl', mode='a') as writer:
+    merged_df = pd.merge(users_df, audit_df, left_on="user_id", right_on="kenna_user_id", how="inner")
+    merged_df.to_excel(writer, sheet_name='Audit Logs')
 
-    # Save the workbook
-    wb.save('cvm_user_audit.xlsx')
 
 print('User, role, and audit log data has been saved to the file cvm_user_audit.xlsx.')
 print('Users that have never logged in are highlighted in red. Users that have not logged in for over 30 days are highlighted in yellow.')

User Audit/useraudit.py

@@ -14,7 +14,6 @@ def flatten_json(nested_json, exclude=['roles']):
             The flattened json object if successful, None otherwise.
     """
     out = {}
-
     def flatten(x, name='', exclude=exclude):
         if type(x) is dict:
             for a in x:
@@ -31,27 +30,41 @@ def flatten(x, name='', exclude=exclude):
     return out
 
 token = sys.argv[1]
-base_url = "https://api.kennasecurity.com"
-users_url= base_url + "/users"
+per_page = 500
+base_url = "http://api.kennasecurity.com"
+users_url= base_url + "/users?per_page=" + str(per_page)
 roles_url = base_url + "/roles"
 
 #print(users_url)
 
-headers = {"Accept": "application/json", "X-Risk-Token":token, "User-Agent": 'user_audit/1.0.0 (Kenna Security)'}
-
-users_response = requests.get(users_url, headers=headers).json()
+headers = {"Accept": "*/*", "X-Risk-Token":token, "User-Agent": 'PostmanRuntime/7.42.0'}
 
 #print(users_response)
+writer = pd.ExcelWriter('kenna_user_audit.xlsx', engine='xlsxwriter', date_format='m/d/yyyy')
+pages = -1
+page = 1
+users_df = pd.DataFrame()
+
+while True:
+    paged_url = users_url + "&page=" + str(page)
+    print("Requesting data from", paged_url)
+    users_response = requests.get(paged_url, headers=headers).json()
+    users_df = pd.concat([users_df, pd.DataFrame(json_normalize([flatten_json(x) for x in users_response['users']]))], ignore_index=True)   
+    # do this once
+    if 'meta' in users_response and pages == -1:
+        pages = users_response['meta']['pages']
+    if page >= pages:
+        break
+    page += 1
 
-users_df = pd.DataFrame(json_normalize([flatten_json(x) for x in users_response['users']]))
 users_df = users_df.rename(columns={"id":"user_id","created_at":"user_created_at","updated_at":"user_updated_at"})
 
 users_df['user_created_at'] = pd.to_datetime(users_df['user_created_at'], format='%Y-%m-%d', errors='coerce').dt.date
 users_df['last_sign_in_at'] = pd.to_datetime(users_df['last_sign_in_at'], format='%Y-%m-%d', errors='coerce').dt.date
 
 print('Printing Users data sample:')
 print(users_df.head(2))
-
+users_df.to_excel(writer, sheet_name='Users')
 #remove comments to troubleshoot columns
 #for col in users_df.columns:
     #print(col)
@@ -70,11 +83,8 @@ def flatten(x, name='', exclude=exclude):
 #for col in roles_df.columns:
     #print(col)
 
-writer = pd.ExcelWriter('kenna_user_audit.xlsx', engine='xlsxwriter', date_format='m/d/yyyy')
-
 workbook = writer.book
 
-users_df.to_excel(writer, sheet_name='Users')
 roles_df.to_excel(writer, sheet_name='Roles')
 
 red_format = workbook.add_format({'bg_color': '#FFC7CE','font_color': '#9C0006'})
@@ -89,7 +99,7 @@ def flatten(x, name='', exclude=exclude):
 users_sheet.conditional_format('$J$2:$J$99999', {'type': 'blanks', 'format': red_format})
 users_sheet.conditional_format('$J$2:$J$99999', {'type': 'formula', 'criteria': '=J2<TODAY()-30', 'format': yellow_format})
 
-writer.save()
+writer.close()
 
 print('User and role data has been saved to the file kenna_user_audit.xlsx.')
 print('Users that have never logged in are highlighted in red. Users that have not logged in for over 30 days are highlighted in yellow.')