golang: add test against false negatives

Check against the list from https://github.com/microlinkhq/top-user-agents

Fix #350

validate_test.go

@@ -1,7 +1,9 @@
 package agents
 
 import (
+	"encoding/json"
 	"fmt"
+	"net/http"
 	"testing"
 )
 
@@ -48,6 +50,35 @@ func TestPatterns(t *testing.T) {
 	}
 }
 
+func TestFalseNegatives(t *testing.T) {
+	const browsersURL = "https://raw.githubusercontent.com/microlinkhq/top-user-agents/master/src/index.json"
+	resp, err := http.Get(browsersURL)
+	if err != nil {
+		t.Fatalf("Failed to fetch the list of browser User Agents from %s: %v.", browsersURL, err)
+	}
+
+	t.Cleanup(func() {
+		if err := resp.Body.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	var browsers []string
+	if err := json.NewDecoder(resp.Body).Decode(&browsers); err != nil {
+		t.Fatalf("Failed to parse the list of browser User Agents: %v.", err)
+	}
+
+	for _, userAgent := range browsers {
+		if IsCrawler(userAgent) {
+			t.Errorf("Browser User Agent %q is recognized as a crawler.", userAgent)
+		}
+		indices := MatchingCrawlers(userAgent)
+		if len(indices) != 0 {
+			t.Errorf("Browser User Agent %q matches with crawlers %v.", userAgent, indices)
+		}
+	}
+}
+
 const (
 	crawlerUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google (+https://developers.google.com/+/web/snippet/"
 	browserUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) obsidian/1.5.3 Chrome/114.0.5735.289 Electron/25.8.1 Safari/537.36"