Fix twitter post check (#214)

ice-blockchain · Aug 22, 2024 · 12268ec · 12268ec
1 parent 65bb9c5
commit 12268ec
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 12 deletions.
diff --git a/go.mod b/go.mod
@@ -170,7 +170,7 @@ require (
 	golang.org/x/text v0.17.0 // indirect
 	golang.org/x/time v0.6.0 // indirect
 	golang.org/x/tools v0.24.0 // indirect
-	google.golang.org/api v0.193.0 // indirect
+	google.golang.org/api v0.194.0 // indirect
 	google.golang.org/appengine/v2 v2.0.6 // indirect
 	google.golang.org/genproto v0.0.0-20240820151423-278611b39280 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240820151423-278611b39280 // indirect

diff --git a/go.sum b/go.sum
@@ -526,8 +526,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.193.0 h1:eOGDoJFsLU+HpCBaDJex2fWiYujAw9KbXgpOAMePoUs=
-google.golang.org/api v0.193.0/go.mod h1:Po3YMV1XZx+mTku3cfJrlIYR03wiGrCOsdpC67hjZvw=
+google.golang.org/api v0.194.0 h1:dztZKG9HgtIpbI35FhfuSNR/zmaMVdxNlntHj1sIS4s=
+google.golang.org/api v0.194.0/go.mod h1:AgvUFdojGANh3vI+P7EVnxj3AISHllxGCJSFmggmnd0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine/v2 v2.0.6 h1:LvPZLGuchSBslPBp+LAhihBeGSiRh1myRoYK4NtuBIw=

diff --git a/kyc/social/internal/twitter.go b/kyc/social/internal/twitter.go
@@ -36,23 +36,33 @@ func (t *twitterVerifierImpl) VerifyText(ctx context.Context, doc *goquery.Docum
 	return
 }
 
+func compareLinkWithoutRootDomain(target, expectedURL string) bool {
+	domains := []string{"twitter.com", "x.com"}
+
+	for _, domain := range domains {
+		x := "https://" + domain
+		if strings.HasPrefix(target, x) {
+			target = strings.Replace(target, x, "", 1)
+		}
+		if strings.HasPrefix(expectedURL, x) {
+			expectedURL = strings.Replace(expectedURL, x, "", 1)
+		}
+	}
+
+	return strings.EqualFold(target, expectedURL)
+}
+
 func (t *twitterVerifierImpl) VerifyPostLinkOf(ctx context.Context, target, expectedURL string) bool {
 	if strings.EqualFold(target, expectedURL) {
 		return true
 	}
 
 	if strings.HasPrefix(target, "https://t.co") {
 		loc, err := t.Scraper.Fetcher().Head(ctx, target)
-		if strings.HasPrefix(loc, "https://twitter.com") {
-			loc = strings.Replace(loc, "https://twitter.com", "", 1)
-		}
-		if strings.HasPrefix(loc, "https://x.com") {
-			loc = strings.Replace(loc, "https://x.com", "", 1)
-		}
 		if err != nil {
-			log.Warn("twitter: failed to fetch location header", "error", err)
+			log.Warn("twitter: failed to fetch location header: %v", err)
 			// Fallthrough.
-		} else if strings.EqualFold(loc, expectedURL) {
+		} else if compareLinkWithoutRootDomain(loc, expectedURL) {
 			return true
 		}
 	}
@@ -67,11 +77,15 @@ func (t *twitterVerifierImpl) VerifyPostLinkOf(ctx context.Context, target, expe
 	return strings.Contains(strings.ToLower(string(result.Content)), strings.ToLower(expectedURL))
 }
 
+func (*twitterVerifierImpl) IsHashTagLink(target string) bool {
+	return strings.HasPrefix(target, "https://twitter.com/hashtag/") || strings.HasPrefix(target, "https://x.com/hashtag/")
+}
+
 func (t *twitterVerifierImpl) VerifyPostLink(ctx context.Context, doc *goquery.Document, expectedPostURL string) (foundPost bool) {
 	doc.Find("a").EachWithBreak(func(_ int, s *goquery.Selection) bool {
 		for _, node := range s.Nodes {
 			for attrIndex := range node.Attr {
-				if node.Attr[attrIndex].Key == "href" {
+				if node.Attr[attrIndex].Key == "href" && !t.IsHashTagLink(node.Attr[attrIndex].Val) {
 					foundPost = t.VerifyPostLinkOf(ctx, node.Attr[attrIndex].Val, expectedPostURL)
 				}
 			}