diff --git a/application.yaml b/application.yaml index 7749647..8bdd68b 100644 --- a/application.yaml +++ b/application.yaml @@ -51,27 +51,21 @@ kyc/social: max-attempts-allowed: 3 wintr/connectors/storage/v2: *db web-scraping-api: - url-v1: https://bogus/v1 + base-url: https://scrape.shifter.io/ api-key-v1: bogus - url-v2: https://bogus/v2 api-key-v2: bogus + countries: + - MX + - CA + - US social-links: twitter: domains: - twitter.com - x.com - t.co - countries: - - MX - - CA - - US facebook: allow-long-live-tokens: true - cmc: - countries: - - MX - - CA - - US tenant-name: sunwaves referralInviteUrlPrefix: https://bogus.com/@ kyc/face: diff --git a/go.mod b/go.mod index b20092a..647f83d 100644 --- a/go.mod +++ b/go.mod @@ -89,7 +89,7 @@ require ( github.com/go-openapi/swag v0.23.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.22.1 // indirect + github.com/go-playground/validator/v10 v10.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/goccy/go-reflect v1.2.0 // indirect github.com/gofrs/flock v0.12.1 // indirect @@ -130,7 +130,7 @@ require ( github.com/onsi/ginkgo/v2 v2.21.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/opencontainers/runc v1.2.1 // indirect + github.com/opencontainers/runc v1.2.2 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240917153116-6f2963f01587 // indirect diff --git a/go.sum b/go.sum index aebbcf0..a2f46bb 100644 --- a/go.sum +++ b/go.sum @@ -172,8 +172,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= -github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= +github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= @@ -332,8 +332,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opencontainers/runc v1.2.1 h1:mQkmeFSUxqFaVmvIn1VQPeQIKpHFya5R07aJw0DKQa8= -github.com/opencontainers/runc v1.2.1/go.mod h1:/PXzF0h531HTMsYQnmxXkBD7YaGShm/2zcRB79dksUc= +github.com/opencontainers/runc v1.2.2 h1:jTg3Vw2A5f0N9PoxFTEwUhvpANGaNPT3689Yfd/zaX0= +github.com/opencontainers/runc v1.2.2/go.mod h1:/PXzF0h531HTMsYQnmxXkBD7YaGShm/2zcRB79dksUc= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/kyc/scraper/contract.go b/kyc/scraper/contract.go index 34495eb..031b1ac 100644 --- a/kyc/scraper/contract.go +++ b/kyc/scraper/contract.go @@ -13,6 +13,8 @@ import ( const ( applicationYAMLKey = "kyc/social" + scraperV1Suffix = "v1" + scraperV2Suffix = "v2" ) type ( @@ -95,11 +97,7 @@ type ( } configTwitter struct { - Domains []string `yaml:"domains" mapstructure:"domains"` - Countries []string `yaml:"countries" mapstructure:"countries"` - } - configCMC struct { - Countries []string `yaml:"countries" mapstructure:"countries"` + Domains []string `yaml:"domains" mapstructure:"domains"` } configFacebook struct { @@ -110,16 +108,15 @@ type ( config struct { WebScrapingAPI struct { - APIKeyV1 string `yaml:"api-key-v1" mapstructure:"api-key-v1"` //nolint:tagliatelle // Nope. - URLV1 string `yaml:"url-v1" mapstructure:"url-v1"` //nolint:tagliatelle // Nope. - APIKeyV2 string `yaml:"api-key-v2" mapstructure:"api-key-v2"` //nolint:tagliatelle // Nope. - URLV2 string `yaml:"url-v2" mapstructure:"url-v2"` //nolint:tagliatelle // Nope. + APIKeyV1 string `yaml:"api-key-v1" mapstructure:"api-key-v1"` //nolint:tagliatelle // Nope. + APIKeyV2 string `yaml:"api-key-v2" mapstructure:"api-key-v2"` //nolint:tagliatelle // Nope. + BaseURL string `yaml:"base-url" mapstructure:"base-url"` //nolint:tagliatelle // Nope. + Countries []string `yaml:"countries" mapstructure:"countries"` } `yaml:"web-scraping-api" mapstructure:"web-scraping-api"` //nolint:tagliatelle // Nope. SocialLinks struct { Facebook configFacebook `yaml:"facebook" mapstructure:"facebook"` Twitter configTwitter `yaml:"twitter" mapstructure:"twitter"` - CMC configCMC `yaml:"cmc" mapstructure:"cmc"` } `yaml:"social-links" mapstructure:"social-links"` //nolint:tagliatelle // Nope. } diff --git a/kyc/scraper/social.go b/kyc/scraper/social.go index c4b9477..eb4fad2 100644 --- a/kyc/scraper/social.go +++ b/kyc/scraper/social.go @@ -3,7 +3,9 @@ package scraper import ( + "fmt" "os" + "strings" appcfg "github.com/ice-blockchain/wintr/config" "github.com/ice-blockchain/wintr/log" @@ -16,9 +18,8 @@ func loadConfig() *config { for ptr, env := range map[*string]string{ &cfg.WebScrapingAPI.APIKeyV1: os.Getenv("WEB_SCRAPING_API_KEY_V1"), - &cfg.WebScrapingAPI.URLV1: os.Getenv("WEB_SCRAPING_API_URL_V1"), &cfg.WebScrapingAPI.APIKeyV2: os.Getenv("WEB_SCRAPING_API_KEY_V2"), - &cfg.WebScrapingAPI.URLV2: os.Getenv("WEB_SCRAPING_API_URL_V2"), + &cfg.WebScrapingAPI.BaseURL: os.Getenv("WEB_SCRAPING_API_BASE_URL"), &cfg.SocialLinks.Facebook.AppID: os.Getenv("FACEBOOK_APP_ID"), &cfg.SocialLinks.Facebook.AppSecret: os.Getenv("FACEBOOK_APP_SECRET"), } { @@ -32,12 +33,13 @@ func loadConfig() *config { func New(st StrategyType) Verifier { conf := loadConfig() + conf.WebScrapingAPI.BaseURL = strings.TrimSuffix(conf.WebScrapingAPI.BaseURL, "/") switch st { case StrategyTwitter: - sc := newMustWebScraper(conf.WebScrapingAPI.URLV2, conf.WebScrapingAPI.APIKeyV2) + sc := newMustWebScraper(fmt.Sprintf("%v/%v", conf.WebScrapingAPI.BaseURL, scraperV2Suffix), conf.WebScrapingAPI.APIKeyV2) - return newTwitterVerifier(sc, conf.SocialLinks.Twitter.Domains, conf.SocialLinks.Twitter.Countries) + return newTwitterVerifier(sc, conf.SocialLinks.Twitter.Domains, conf.WebScrapingAPI.Countries) case StrategyFacebook: sc := new(dataFetcherImpl) @@ -50,9 +52,9 @@ func New(st StrategyType) Verifier { ) case StrategyCMC: - sc := newMustWebScraper(conf.WebScrapingAPI.URLV1, conf.WebScrapingAPI.APIKeyV1) + sc := newMustWebScraper(fmt.Sprintf("%v/%v", conf.WebScrapingAPI.BaseURL, scraperV1Suffix), conf.WebScrapingAPI.APIKeyV1) - return newCMCVerifier(sc, conf.SocialLinks.CMC.Countries) + return newCMCVerifier(sc, conf.WebScrapingAPI.Countries) default: log.Panic("invalid social verifier: " + st) } diff --git a/kyc/scraper/social_test.go b/kyc/scraper/social_test.go index 3f37ffd..c610aa2 100644 --- a/kyc/scraper/social_test.go +++ b/kyc/scraper/social_test.go @@ -4,6 +4,7 @@ package scraper import ( "context" + "fmt" "os" "testing" "time" @@ -23,7 +24,7 @@ func TestTwitterKYC(t *testing.T) { conf := loadConfig() require.NotNil(t, conf) - sc := newMustWebScraper(conf.WebScrapingAPI.URLV2, conf.WebScrapingAPI.APIKeyV2) + sc := newMustWebScraper(fmt.Sprintf("%v/%v", conf.WebScrapingAPI.BaseURL, scraperV2Suffix), conf.WebScrapingAPI.APIKeyV2) require.NotNil(t, sc) verifier := newTwitterVerifier(sc, []string{"twitter.com"}, []string{"US", "MX", "CA"}) @@ -53,7 +54,7 @@ func TestTwitterLinkInPostKYC(t *testing.T) { conf := loadConfig() require.NotNil(t, conf) - sc := newMustWebScraper(conf.WebScrapingAPI.URLV2, conf.WebScrapingAPI.APIKeyV2) + sc := newMustWebScraper(fmt.Sprintf("%v/%v", conf.WebScrapingAPI.BaseURL, scraperV2Suffix), conf.WebScrapingAPI.APIKeyV2) require.NotNil(t, sc) verifier := newTwitterVerifier(sc, []string{"x.com"}, []string{"US", "MX", "CA"}) @@ -78,7 +79,7 @@ func TestTwitterKYCNoRepost(t *testing.T) { conf := loadConfig() require.NotNil(t, conf) - sc := newMustWebScraper(conf.WebScrapingAPI.URLV2, conf.WebScrapingAPI.APIKeyV2) + sc := newMustWebScraper(fmt.Sprintf("%v/%v", conf.WebScrapingAPI.BaseURL, scraperV2Suffix), conf.WebScrapingAPI.APIKeyV2) require.NotNil(t, sc) verifier := newTwitterVerifier(sc, []string{"twitter.com"}, []string{"US", "MX", "CA"}) @@ -99,7 +100,7 @@ func TestTwitterPrivate(t *testing.T) { conf := loadConfig() require.NotNil(t, conf) - sc := newMustWebScraper(conf.WebScrapingAPI.URLV2, conf.WebScrapingAPI.APIKeyV2) + sc := newMustWebScraper(fmt.Sprintf("%v/%v", conf.WebScrapingAPI.BaseURL, scraperV2Suffix), conf.WebScrapingAPI.APIKeyV2) require.NotNil(t, sc) verifier := newTwitterVerifier(sc, []string{"twitter.com"}, []string{"US", "MX", "CA"})