diff --git a/verify-loader b/verify-loader index e05af70..59bf6f8 100755 --- a/verify-loader +++ b/verify-loader @@ -111,7 +111,7 @@ def print_stats(invocid, ctx, payload): assert ctx.prev == statseq assert timestamp[-1] == 's' now = time.time() - print("%s: %.2fs (%5.2fs) %12.3f %12.3f %d %d %d %d" % ( + print("{}: {:.2f}s ({:5.2f}s) {:12.3f} {:12.3f} {} {} {} {}".format( invocid, ts, now - ts, lclrate, gblrate, statseq, ctx.count, ctx.skips, ctx.duplicates)) sys.stdout.flush() @@ -122,8 +122,10 @@ def verify(input_gen, report_interval=REPORT_INTERVAL): ignored_bytes = 0 ignored_count = 0 + out_of_seq = 0 report_bytes = 0 + # report stats after every MB report_bytes_target = report_interval * MB report_ignored_bytes = 0 report_ignored_count = 0 @@ -135,16 +137,19 @@ def verify(input_gen, report_interval=REPORT_INTERVAL): try: for line in input_gen: line_len = len(line) + #find the log header: loader seq - if not line.startswith("loader seq - "): report_ignored_bytes += line_len report_ignored_count += 1 else: + #check that the line has all the constituent parts (
- - - ) try: _, invocid, seqval, payload = line.split('-', 4) except Exception: report_ignored_bytes += line_len report_ignored_count += 1 else: + #check if seq_num is valid try: seq = int(seqval) except Exception: @@ -155,9 +160,11 @@ def verify(input_gen, report_interval=REPORT_INTERVAL): invocid = invocid.strip() ctx = contexts[invocid] prev = ctx.msg(seq, line_len) + # check for out of order lines - this implies LOG LOSS if prev is not None: # Bad record encountered, flag it - print("%s: %d %d <-" % (invocid, seq, prev)) + print("{}: {} {} <-".format(invocid, seq, prev)) + out_of_seq += 1 sys.stdout.flush() if payload.startswith(" (stats:"): print_stats(invocid, ctx, payload) @@ -176,14 +183,16 @@ def verify(input_gen, report_interval=REPORT_INTERVAL): for invocid, ctx in contexts.items(): report_count += ctx.report_count if ctx.report(): - print("%s: %d %d %d" % (invocid, ctx.count, ctx.skips, ctx.duplicates)) + print("{}: {} {} {}".format(invocid, ctx.count, ctx.skips, ctx.duplicates)) total_bytes += ctx.bytes total_count += ctx.count - print("interval read rate: %.3f MB/sec, %.3f/sec " \ - "(ignored %.3f MB/sec, %.3f/sec); " \ - "overall read rate: %.3f MB/sec %.3f/sec " \ - "(ignored %.3f MB/sec, %.3f/sec)" % ( + print("interval read rate: {%.3f} MB/sec, {%.3f}/sec " \ + "(ignored {%.3f} MB/sec, {%.3f}/sec); " \ + "overall read rate: {%.3f} MB/sec {%.3f}/sec " \ + "(ignored {%.3f} MB/sec, {%.3f}/sec)" \ + "(total lines {})" \ + "(ignored lines {})".format( (report_bytes / MB) / (now - report_start), (report_count / (now - report_start)), (report_ignored_bytes / MB) / (now - report_start), @@ -191,7 +200,8 @@ def verify(input_gen, report_interval=REPORT_INTERVAL): (total_bytes / MB) / (now - start), (total_count / (now - start)), (ignored_bytes / MB) / (now - start), - (ignored_count / (now - start)))) + (ignored_count / (now - start)), + (report_count, report_ignored_count))) print("--- verify-loader\n") sys.stdout.flush() @@ -219,13 +229,16 @@ def verify(input_gen, report_interval=REPORT_INTERVAL): if ignored_count + total_count > 0: print("\n+++ verify-loader") for invocid, ctx in contexts.items(): - print("%s: %d %d %d" % (invocid, ctx.count, ctx.skips, ctx.duplicates)) - print("overall read rate: %.3f MB/sec %.3f/sec " \ - "(ignored %.3f MB/sec, %.3f/sec)" % ( + print("{}: {} {} {}".format(invocid, ctx.count, ctx.skips, ctx.duplicates)) + + print("overall read rate: {:.3f} MB/sec {:.3f}/sec " \ + "(ignored {:.3f} MB/sec, {:.3f}/sec)" \ + "(total bytes {}, total lines {}, ignored lines {}, bad sequence {}({:.3f}%))".format( (total_bytes / MB) / (now - start), (total_count / (now - start)), (ignored_bytes / MB) / (now - start), - (ignored_count / (now - start)))) + (ignored_count / (now - start)), + total_bytes, total_count, ignored_count, out_of_seq, (out_of_seq*100.0)/total_count)) print("--- verify-loader\n") if tot_skips + tot_dupes > 0: ret_val = 1