-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' of https://github.com/eosnetworkfoundation/replay…
- Loading branch information
Showing
7 changed files
with
111 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| """Parse Orchestration File and Calculate Job Elapsed Time""" | ||
| import argparse | ||
| from datetime import datetime | ||
| import statistics | ||
| import numpy as np | ||
|
|
||
| if __name__ == '__main__': | ||
| parser = argparse.ArgumentParser( | ||
| description='helper script to extract elapsed timing from log') | ||
| parser.add_argument('--log', type=str, help='path to config file') | ||
|
|
||
| args = parser.parse_args() | ||
| timings = [] | ||
|
|
||
| # Open the file and read log_entry by log_entry | ||
| with open(args.log, 'r', encoding='utf-8') as file: | ||
| for log_entry in file: | ||
| # Check if the specific phrase is in the current log_entry | ||
| if "OrchWebSrv INFO Completed Job" in log_entry: | ||
| complete_record = {} | ||
| # Print the log_entry or perform other actions | ||
| for part in log_entry.split(','): | ||
| if 'starttime' in part: | ||
| starttimestr = part.split(': ', 1)[1] | ||
| complete_record['starttime'] = datetime.strptime( | ||
| starttimestr, '%Y-%m-%dT%H:%M:%S') | ||
| elif 'endtime' in part: | ||
| endtimestr = part.split(': ', 1)[1] | ||
| complete_record['endtime'] = datetime.strptime( | ||
| endtimestr, '%Y-%m-%dT%H:%M:%S') | ||
| elif 'jobid' in part: | ||
| complete_record['jobid'] = part.split(': ', 1)[1].strip() | ||
| elif 'config' in part: | ||
| complete_record['config'] = part.split(': ', 1)[1].strip() | ||
| elif 'snapshot' in part: | ||
| complete_record['snapshot'] = part.split(': ', 1)[1].strip() | ||
| # calc elapsed time | ||
| timedelta = complete_record['endtime'] - complete_record['starttime'] | ||
| # Convert the difference to total minutes | ||
| complete_record['total_minutes'] = int(timedelta.total_seconds())/60 | ||
| timings.append(complete_record) | ||
|
|
||
| # Calculate average (mean) | ||
| average = statistics.mean(list(record['total_minutes'] for record in timings)) | ||
|
|
||
| # Calculate standard deviation | ||
| std_dev = statistics.stdev(list(record['total_minutes'] for record in timings)) | ||
|
|
||
| # Calculate median | ||
| median = statistics.median(list(record['total_minutes'] for record in timings)) | ||
|
|
||
| # Calculate the 75th and 90th percentiles | ||
| percentile_75 = np.percentile(list(record['total_minutes'] for record in timings), 75) | ||
| percentile_90 = np.percentile(list(record['total_minutes'] for record in timings), 90) | ||
|
|
||
| # get longest | ||
| longest = max(list(record['total_minutes'] for record in timings)) | ||
|
|
||
| # Print the results | ||
| print("JOB TIMING ALL TIMES IN MINUTES") | ||
| print("-------------------------------") | ||
| print(f"Number of Jobs: {len(timings)}") | ||
| print(f"Average: {round(average,2)}") | ||
| print(f"Standard Deviation: {round(std_dev,2)}") | ||
| print(f"Median: {round(median,2)}") | ||
| print(f"75th Percentile: {round(percentile_75,2)}") | ||
| print(f"90th Percentile: {round(percentile_90,2)}") | ||
| print(f"Longest Running Job {round(longest,2)} mins") | ||
|
|
||
| if std_dev > average: | ||
| print("\nLONG RUNNING JOBS TOP 90%") | ||
| print("-------------------------") | ||
| for record in timings: | ||
| if record['total_minutes'] > percentile_90: | ||
| # when config and snapshot exist print full | ||
| # field check for backwards compat | ||
| if 'config' in record and 'snapshot' in record \ | ||
| and record['config'] and record['snapshot']: | ||
| print(f"Job {record['jobid']}\ | ||
| running time {round(record['total_minutes'],2)}\ | ||
| config {record['config']} with snapshot {record['snapshot']}") | ||
| else: | ||
| print(f"Job {record['jobid']}\ | ||
| running time {round(record['total_minutes'],2)}") |