(fix): html-report ordering was still flaky, should now be robust #190

Examples being the culprit, in large specs with lots of examples, some ordering was failing because there was not enough detail in the sort (it was only checking a couple of properties). Now sorting checks all the way down the model and applies a hash to paths and messages to ensure consistent ordering.

Testing against stripe, k8s, petstore and user supplied specs that were causing issues.

Signed-off-by: Dave Shanley <[email protected]>

html-report/build_report.go

@@ -4,6 +4,7 @@ package html_report
 
 import (
 	"bytes"
+	"crypto/sha256"
 	_ "embed"
 	"encoding/json"
 	"fmt"
@@ -89,7 +90,40 @@ func (html htmlReport) GenerateReport(test bool) []byte {
 		},
 		"sortResults": func(results []*model.RuleFunctionResult) []*model.RuleFunctionResult {
 			sort.Slice(results, func(i, j int) bool {
-				return results[i].Range.Start.Line < results[j].Range.Start.Line
+				if results[i].Range.Start.Line < results[j].Range.Start.Line {
+					return true
+				}
+				if results[i].Range.Start.Line > results[j].Range.Start.Line {
+					return false
+				}
+				if results[i].Range.Start.Char < results[j].Range.Start.Char {
+					return true
+				}
+				if results[i].Range.Start.Char > results[j].Range.Start.Char {
+					return false
+				}
+				if results[i].Range.End.Line < results[j].Range.End.Line {
+					return true
+				}
+				if results[i].Range.End.Line > results[j].Range.End.Line {
+					return false
+				}
+				if results[i].Range.End.Char < results[j].Range.End.Char {
+					return true
+				}
+				if results[i].Range.End.Char > results[j].Range.End.Char {
+					return false
+				}
+				if results[i].Path != results[j].Path {
+					// sha256 these paths for consistency
+					lm := fmt.Sprintf("%x", sha256.Sum256([]byte(results[i].Path)))
+					rm := fmt.Sprintf("%x", sha256.Sum256([]byte(results[j].Path)))
+					return lm < rm
+				}
+				// sha256 these paths for consistency
+				lm := fmt.Sprintf("%x", sha256.Sum256([]byte(results[i].Message)))
+				rm := fmt.Sprintf("%x", sha256.Sum256([]byte(results[j].Message)))
+				return lm < rm
 			})
 			return results
 		},
@@ -102,7 +136,13 @@ func (html htmlReport) GenerateReport(test bool) []byte {
 
 			r = results.GetResultsForCategoryWithLimit(cat, limit)
 			sort.Slice(r.RuleResults, func(i, j int) bool {
-				return r.RuleResults[i].Rule.Id < r.RuleResults[j].Rule.Id
+				if r.RuleResults[i].Rule.Id < r.RuleResults[j].Rule.Id {
+					return true
+				}
+				if r.RuleResults[i].Rule.Id > r.RuleResults[j].Rule.Id {
+					return false
+				}
+				return true
 			})
 			return r
 		},

model/results.go

@@ -375,8 +375,14 @@ func (rr *RuleResultSet) Len() int { return len(rr.Results) }
 
 // Less determines which result has the lower line number
 func (rr *RuleResultSet) Less(i, j int) bool {
-	if rr.Results != nil && rr.Results[i].StartNode != nil && rr.Results[j].StartNode != nil {
-		return rr.Results[i].StartNode.Line < rr.Results[j].StartNode.Line
+	if rr.Results[i].StartNode != nil && rr.Results[j].StartNode != nil {
+		if rr.Results[i].StartNode.Line < rr.Results[j].StartNode.Line {
+			return true
+		}
+		if rr.Results[i].StartNode.Line > rr.Results[j].StartNode.Line {
+			return false
+		}
+		return rr.Results[i].RuleId < rr.Results[j].RuleId
 	}
 	return false
 }