From 96af4b4a29e3725378a6ec8619ad8bc04be3f300 Mon Sep 17 00:00:00 2001 From: Myles <96409608+ice-myles@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:03:58 +0300 Subject: [PATCH] CMC verification changes (#237) --- go.mod | 4 +- go.sum | 8 +-- kyc/scraper/cmc.go | 58 +++++++++++-------- kyc/scraper/contract.go | 6 ++ kyc/verification_scenarios/contract.go | 3 +- .../verification_scenarios.go | 9 ++- 6 files changed, 50 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index 32ac542..5e3a639 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( cloud.google.com/go/compute/metadata v0.5.2 // indirect cloud.google.com/go/firestore v1.17.0 // indirect cloud.google.com/go/iam v1.2.2 // indirect - cloud.google.com/go/longrunning v0.6.2 // indirect + cloud.google.com/go/longrunning v0.6.3 // indirect cloud.google.com/go/monitoring v1.21.2 // indirect cloud.google.com/go/storage v1.47.0 // indirect firebase.google.com/go/v4 v4.15.0 // indirect @@ -184,7 +184,7 @@ require ( golang.org/x/text v0.20.0 // indirect golang.org/x/time v0.8.0 // indirect golang.org/x/tools v0.27.0 // indirect - google.golang.org/api v0.206.0 // indirect + google.golang.org/api v0.207.0 // indirect google.golang.org/appengine/v2 v2.0.6 // indirect google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect diff --git a/go.sum b/go.sum index 1458760..aeb6ffb 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,8 @@ cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA= cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY= cloud.google.com/go/logging v1.12.0 h1:ex1igYcGFd4S/RZWOCU51StlIEuey5bjqwH9ZYjHibk= cloud.google.com/go/logging v1.12.0/go.mod h1:wwYBt5HlYP1InnrtYI0wtwttpVU1rifnMT7RejksUAM= -cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc= -cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= +cloud.google.com/go/longrunning v0.6.3 h1:A2q2vuyXysRcwzqDpMMLSI6mb6o39miS52UEG/Rd2ng= +cloud.google.com/go/longrunning v0.6.3/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= cloud.google.com/go/monitoring v1.21.2 h1:FChwVtClH19E7pJ+e0xUhJPGksctZNVOk2UhMmblmdU= cloud.google.com/go/monitoring v1.21.2/go.mod h1:hS3pXvaG8KgWTSz+dAdyzPrGUYmi2Q+WFX8g2hqVEZU= cloud.google.com/go/storage v1.47.0 h1:ajqgt30fnOMmLfWfu1PWcb+V9Dxz6n+9WKjdNg5R4HM= @@ -563,8 +563,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.206.0 h1:A27GClesCSheW5P2BymVHjpEeQ2XHH8DI8Srs2HI2L8= -google.golang.org/api v0.206.0/go.mod h1:BtB8bfjTYIrai3d8UyvPmV9REGgox7coh+ZRwm0b+W8= +google.golang.org/api v0.207.0 h1:Fvt6IGCYjf7YLcQ+GCegeAI2QSQCfIWhRkmrMPj3JRM= +google.golang.org/api v0.207.0/go.mod h1:I53S168Yr/PNDNMi5yPnDc0/LGRZO6o7PoEbl/HY3CM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine/v2 v2.0.6 h1:LvPZLGuchSBslPBp+LAhihBeGSiRh1myRoYK4NtuBIw= diff --git a/kyc/scraper/cmc.go b/kyc/scraper/cmc.go index 1cf3811..e176499 100644 --- a/kyc/scraper/cmc.go +++ b/kyc/scraper/cmc.go @@ -21,47 +21,57 @@ func newCMCVerifier(sc webScraper, countries []string) *cmcVerifierImpl { } func (c *cmcVerifierImpl) VerifyPost(ctx context.Context, meta *Metadata) (username string, err error) { - if !validateProfileURL(meta.PostURL) { + if !validatePostURL(meta.PostURL) { return "", errors.Wrapf(ErrInvalidURL, "invalid url: %v", meta.PostURL) } - postURL := normalizeProfileURL(meta.PostURL) - oe, err := c.Scrape(ctx, postURL) + oe, err := c.Scrape(ctx, meta.PostURL) if err != nil { - return "", errors.Wrapf(err, "can't scrape: %v", postURL) + return "", errors.Wrapf(err, "can't scrape: %v", meta.PostURL) } - return "", errors.Wrapf(verifyFollowing(oe.Content, meta), "can't verify following: %v", postURL) + return "", errors.Wrapf(verifyPost(oe.Content), "can't verify cmc link: %v", meta.PostURL) } -func verifyFollowing(html []byte, meta *Metadata) (err error) { +//nolint:funlen // . +func verifyPost(html []byte) (err error) { doc, err := goquery.NewDocumentFromReader(bytes.NewReader(html)) if err != nil { return multierror.Append(ErrInvalidPageContent, err) } - found := false - doc.Find("div.name div.nickName").EachWithBreak(func(_ int, s *goquery.Selection) bool { - found = strings.EqualFold(s.Find("span").Text(), meta.ExpectedPostText) + var ( + foundIceCoin = false + foundPairedCoin = false + ) + doc.Find("#post-detail a.coin-link").EachWithBreak(func(_ int, s *goquery.Selection) bool { + const coinPrefix = "$" + txt := s.Find("span.real-text").Text() + txt, foundPrefix := strings.CutPrefix(txt, coinPrefix) + if !foundPrefix { + return false + } + if foundIce := strings.EqualFold(txt, iceCoinName); foundIce { + foundIceCoin = true + } else { + for _, coinName := range supportedCMCCoinNameList { + if foundPaired := strings.EqualFold(txt, coinName); foundPaired { + foundPairedCoin = true + + break + } + } + } - return !found + return !foundIceCoin || !foundPairedCoin }) - if !found { - return errors.Wrapf(ErrTextNotFound, "text %v not found", meta.ExpectedPostText) + if !foundIceCoin || !foundPairedCoin { + return errors.Wrap(ErrTextNotFound, "ice coin or paired coin not found") } return nil } -func validateProfileURL(url string) bool { - return strings.HasPrefix(url, "https://coinmarketcap.com/community/profile") -} - -func normalizeProfileURL(url string) string { - res := strings.TrimSuffix(url, "/") - if !strings.HasSuffix(res, "following") { - res += "/following" - } - - return res +func validatePostURL(url string) bool { + return strings.HasPrefix(url, "https://coinmarketcap.com/community/post") } func (c *cmcVerifierImpl) Scrape(ctx context.Context, target string) (result *webScraperResult, err error) { //nolint:funlen // . @@ -76,8 +86,6 @@ func (c *cmcVerifierImpl) Scrape(ctx context.Context, target string) (result *we Retry: twitterRetryFn, Options: func(options map[string]string) map[string]string { options["country"] = country - options["wait_for_css"] = ".user-card__container" - options["auto_solve"] = trueVal options["block_resources"] = falseVal options["load_iframes"] = trueVal options["load_shadowroots"] = falseVal diff --git a/kyc/scraper/contract.go b/kyc/scraper/contract.go index 031b1ac..0f3c01e 100644 --- a/kyc/scraper/contract.go +++ b/kyc/scraper/contract.go @@ -15,6 +15,7 @@ const ( applicationYAMLKey = "kyc/social" scraperV1Suffix = "v1" scraperV2Suffix = "v2" + iceCoinName = "ICE" ) type ( @@ -154,6 +155,7 @@ const ( StrategyCMC StrategyType = "cmc" ) +//nolint:gochecknoglobals // . var ( ErrInvalidPageContent = errors.New("invalid page content") ErrTextNotFound = errors.New("expected text not found") @@ -165,4 +167,8 @@ var ( ErrScrapeFailed = errors.New("cannot scrape target") ErrInvalidToken = errors.New("invalid token") ErrTweetPrivate = errors.New("tweet is private or does not exist") + + supportedCMCCoinNameList = []string{ + "BTC", "ETH", "USDT", "SOL", "BNB", "XRP", "DOGE", "USDC", "ADA", "TRX", "SHIB", "AVAX", "TON", "SUI", "LINK", "DOT", "BCH", "PEPE", "LEO", "XLM", "NEAR", "LTC", "APT", "UNI", "DAI", "CRO", "HBAR", "ICP", "RENDER", "BONK", "KAS", "ETC", "OM", "TAO", "POL", "WIF", "FET", "XMR", "ARB", "STX", "OKB", "VET", "FIL", "ATOM", "MNT", "AAVE", "FDUSD", "INJ", "FLOKI", "IMX", //nolint:lll // . + } ) diff --git a/kyc/verification_scenarios/contract.go b/kyc/verification_scenarios/contract.go index 04b35ed..e55a10d 100644 --- a/kyc/verification_scenarios/contract.go +++ b/kyc/verification_scenarios/contract.go @@ -33,8 +33,7 @@ const ( CoinDistributionScenarioSignUpDoctorx TenantScenario = "signup_doctorx" CoinDistributionScenarioSignUpTokero TenantScenario = "signup_tokero" - singUpPrefix = "signup" - iceOpenNetworkHandle = "IceOpenNetwork" + singUpPrefix = "signup" ) // . diff --git a/kyc/verification_scenarios/verification_scenarios.go b/kyc/verification_scenarios/verification_scenarios.go index 3278d8d..9cdc6c3 100644 --- a/kyc/verification_scenarios/verification_scenarios.go +++ b/kyc/verification_scenarios/verification_scenarios.go @@ -62,7 +62,7 @@ func (r *repository) VerifyScenarios(ctx context.Context, metadata *Verification } switch metadata.ScenarioEnum { case CoinDistributionScenarioCmc: - if vErr := r.VerifyCMCProfile(ctx, metadata); vErr != nil { + if vErr := r.VerifyCMC(ctx, metadata); vErr != nil { return nil, errors.Wrapf(vErr, "haven't passed the CMC verification for userID:%v", metadata.UserID) } case CoinDistributionScenarioTwitter: @@ -393,15 +393,14 @@ func (r *repository) VerifyTwitterPost(ctx context.Context, metadata *Verificati return nil } -func (r *repository) VerifyCMCProfile(ctx context.Context, metadata *VerificationMetadata) error { +func (r *repository) VerifyCMC(ctx context.Context, metadata *VerificationMetadata) error { pvm := &social.Metadata{ - PostURL: metadata.CMCProfileLink, - ExpectedPostText: iceOpenNetworkHandle, + PostURL: metadata.CMCProfileLink, } _, err := r.cmcVerifier.VerifyPost(ctx, pvm) if err != nil { return errors.Wrapf(ErrVerificationNotPassed, - "can't verify post for cmc verifier userID:%v,reason:%v", metadata.UserID, social.DetectReason(err)) + "can't verify post for cmc verifier userID:%v,reason:%v", metadata.UserID, err.Error()) } return nil