Configuration fixes.

application.yaml

@@ -51,27 +51,21 @@ kyc/social:
   max-attempts-allowed: 3
   wintr/connectors/storage/v2: *db
   web-scraping-api:
-    url-v1: https://bogus/v1
+    base-url: https://scrape.shifter.io/
     api-key-v1: bogus
-    url-v2: https://bogus/v2
     api-key-v2: bogus
+    countries:
+      - MX
+      - CA
+      - US
   social-links:
     twitter:
       domains:
         - twitter.com
         - x.com
         - t.co
-      countries:
-        - MX
-        - CA
-        - US
     facebook:
       allow-long-live-tokens: true
-    cmc:
-      countries:
-        - MX
-        - CA
-        - US
   tenant-name: sunwaves
   referralInviteUrlPrefix: https://bogus.com/@
 kyc/face:

go.mod

@@ -89,7 +89,7 @@ require (
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.22.1 // indirect
+	github.com/go-playground/validator/v10 v10.23.0 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/goccy/go-reflect v1.2.0 // indirect
 	github.com/gofrs/flock v0.12.1 // indirect
@@ -130,7 +130,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.21.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
-	github.com/opencontainers/runc v1.2.1 // indirect
+	github.com/opencontainers/runc v1.2.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240917153116-6f2963f01587 // indirect

go.sum

@@ -172,8 +172,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
-github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA=
-github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o=
+github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
@@ -332,8 +332,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
 github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
-github.com/opencontainers/runc v1.2.1 h1:mQkmeFSUxqFaVmvIn1VQPeQIKpHFya5R07aJw0DKQa8=
-github.com/opencontainers/runc v1.2.1/go.mod h1:/PXzF0h531HTMsYQnmxXkBD7YaGShm/2zcRB79dksUc=
+github.com/opencontainers/runc v1.2.2 h1:jTg3Vw2A5f0N9PoxFTEwUhvpANGaNPT3689Yfd/zaX0=
+github.com/opencontainers/runc v1.2.2/go.mod h1:/PXzF0h531HTMsYQnmxXkBD7YaGShm/2zcRB79dksUc=
 github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
 github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=

kyc/scraper/contract.go

@@ -13,6 +13,8 @@ import (
 
 const (
 	applicationYAMLKey = "kyc/social"
+	scraperV1Suffix    = "v1"
+	scraperV2Suffix    = "v2"
 )
 
 type (
@@ -95,11 +97,7 @@ type (
 	}
 
 	configTwitter struct {
-		Domains   []string `yaml:"domains"  mapstructure:"domains"`
-		Countries []string `yaml:"countries"  mapstructure:"countries"`
-	}
-	configCMC struct {
-		Countries []string `yaml:"countries"  mapstructure:"countries"`
+		Domains []string `yaml:"domains"  mapstructure:"domains"`
 	}
 
 	configFacebook struct {
@@ -110,16 +108,15 @@ type (
 
 	config struct {
 		WebScrapingAPI struct {
-			APIKeyV1 string `yaml:"api-key-v1" mapstructure:"api-key-v1"` //nolint:tagliatelle // Nope.
-			URLV1    string `yaml:"url-v1"     mapstructure:"url-v1"`     //nolint:tagliatelle // Nope.
-			APIKeyV2 string `yaml:"api-key-v2" mapstructure:"api-key-v2"` //nolint:tagliatelle // Nope.
-			URLV2    string `yaml:"url-v2"     mapstructure:"url-v2"`     //nolint:tagliatelle // Nope.
+			APIKeyV1  string   `yaml:"api-key-v1" mapstructure:"api-key-v1"` //nolint:tagliatelle // Nope.
+			APIKeyV2  string   `yaml:"api-key-v2" mapstructure:"api-key-v2"` //nolint:tagliatelle // Nope.
+			BaseURL   string   `yaml:"base-url"   mapstructure:"base-url"`   //nolint:tagliatelle // Nope.
+			Countries []string `yaml:"countries"  mapstructure:"countries"`
 		} `yaml:"web-scraping-api" mapstructure:"web-scraping-api"` //nolint:tagliatelle // Nope.
 
 		SocialLinks struct {
 			Facebook configFacebook `yaml:"facebook" mapstructure:"facebook"`
 			Twitter  configTwitter  `yaml:"twitter"  mapstructure:"twitter"`
-			CMC      configCMC      `yaml:"cmc"  mapstructure:"cmc"`
 		} `yaml:"social-links" mapstructure:"social-links"` //nolint:tagliatelle // Nope.
 	}
 

kyc/scraper/social.go

@@ -3,7 +3,9 @@
 package scraper
 
 import (
+	"fmt"
 	"os"
+	"strings"
 
 	appcfg "github.com/ice-blockchain/wintr/config"
 	"github.com/ice-blockchain/wintr/log"
@@ -16,9 +18,8 @@ func loadConfig() *config {
 
 	for ptr, env := range map[*string]string{
 		&cfg.WebScrapingAPI.APIKeyV1:        os.Getenv("WEB_SCRAPING_API_KEY_V1"),
-		&cfg.WebScrapingAPI.URLV1:           os.Getenv("WEB_SCRAPING_API_URL_V1"),
 		&cfg.WebScrapingAPI.APIKeyV2:        os.Getenv("WEB_SCRAPING_API_KEY_V2"),
-		&cfg.WebScrapingAPI.URLV2:           os.Getenv("WEB_SCRAPING_API_URL_V2"),
+		&cfg.WebScrapingAPI.BaseURL:         os.Getenv("WEB_SCRAPING_API_BASE_URL"),
 		&cfg.SocialLinks.Facebook.AppID:     os.Getenv("FACEBOOK_APP_ID"),
 		&cfg.SocialLinks.Facebook.AppSecret: os.Getenv("FACEBOOK_APP_SECRET"),
 	} {
@@ -32,12 +33,13 @@ func loadConfig() *config {
 
 func New(st StrategyType) Verifier {
 	conf := loadConfig()
+	conf.WebScrapingAPI.BaseURL = strings.TrimSuffix(conf.WebScrapingAPI.BaseURL, "/")
 
 	switch st {
 	case StrategyTwitter:
-		sc := newMustWebScraper(conf.WebScrapingAPI.URLV2, conf.WebScrapingAPI.APIKeyV2)
+		sc := newMustWebScraper(fmt.Sprintf("%v/%v", conf.WebScrapingAPI.BaseURL, scraperV2Suffix), conf.WebScrapingAPI.APIKeyV2)
 
-		return newTwitterVerifier(sc, conf.SocialLinks.Twitter.Domains, conf.SocialLinks.Twitter.Countries)
+		return newTwitterVerifier(sc, conf.SocialLinks.Twitter.Domains, conf.WebScrapingAPI.Countries)
 
 	case StrategyFacebook:
 		sc := new(dataFetcherImpl)
@@ -50,9 +52,9 @@ func New(st StrategyType) Verifier {
 		)
 
 	case StrategyCMC:
-		sc := newMustWebScraper(conf.WebScrapingAPI.URLV1, conf.WebScrapingAPI.APIKeyV1)
+		sc := newMustWebScraper(fmt.Sprintf("%v/%v", conf.WebScrapingAPI.BaseURL, scraperV1Suffix), conf.WebScrapingAPI.APIKeyV1)
 
-		return newCMCVerifier(sc, conf.SocialLinks.CMC.Countries)
+		return newCMCVerifier(sc, conf.WebScrapingAPI.Countries)
 	default:
 		log.Panic("invalid social verifier: " + st)
 	}

kyc/scraper/social_test.go

@@ -4,6 +4,7 @@ package scraper
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"testing"
 	"time"
@@ -23,7 +24,7 @@ func TestTwitterKYC(t *testing.T) {
 	conf := loadConfig()
 	require.NotNil(t, conf)
 
-	sc := newMustWebScraper(conf.WebScrapingAPI.URLV2, conf.WebScrapingAPI.APIKeyV2)
+	sc := newMustWebScraper(fmt.Sprintf("%v/%v", conf.WebScrapingAPI.BaseURL, scraperV2Suffix), conf.WebScrapingAPI.APIKeyV2)
 	require.NotNil(t, sc)
 
 	verifier := newTwitterVerifier(sc, []string{"twitter.com"}, []string{"US", "MX", "CA"})
@@ -53,7 +54,7 @@ func TestTwitterLinkInPostKYC(t *testing.T) {
 	conf := loadConfig()
 	require.NotNil(t, conf)
 
-	sc := newMustWebScraper(conf.WebScrapingAPI.URLV2, conf.WebScrapingAPI.APIKeyV2)
+	sc := newMustWebScraper(fmt.Sprintf("%v/%v", conf.WebScrapingAPI.BaseURL, scraperV2Suffix), conf.WebScrapingAPI.APIKeyV2)
 	require.NotNil(t, sc)
 
 	verifier := newTwitterVerifier(sc, []string{"x.com"}, []string{"US", "MX", "CA"})
@@ -78,7 +79,7 @@ func TestTwitterKYCNoRepost(t *testing.T) {
 	conf := loadConfig()
 	require.NotNil(t, conf)
 
-	sc := newMustWebScraper(conf.WebScrapingAPI.URLV2, conf.WebScrapingAPI.APIKeyV2)
+	sc := newMustWebScraper(fmt.Sprintf("%v/%v", conf.WebScrapingAPI.BaseURL, scraperV2Suffix), conf.WebScrapingAPI.APIKeyV2)
 	require.NotNil(t, sc)
 
 	verifier := newTwitterVerifier(sc, []string{"twitter.com"}, []string{"US", "MX", "CA"})
@@ -99,7 +100,7 @@ func TestTwitterPrivate(t *testing.T) {
 	conf := loadConfig()
 	require.NotNil(t, conf)
 
-	sc := newMustWebScraper(conf.WebScrapingAPI.URLV2, conf.WebScrapingAPI.APIKeyV2)
+	sc := newMustWebScraper(fmt.Sprintf("%v/%v", conf.WebScrapingAPI.BaseURL, scraperV2Suffix), conf.WebScrapingAPI.APIKeyV2)
 	require.NotNil(t, sc)
 
 	verifier := newTwitterVerifier(sc, []string{"twitter.com"}, []string{"US", "MX", "CA"})