Skip to content

Commit

Permalink
Merge pull request #141 from isucon/bench/impl-24
Browse files Browse the repository at this point in the history
bench 高速化したappへの負荷を調整する編
  • Loading branch information
ToshihitoKon authored Jul 5, 2022
2 parents a12e3e0 + 5f4e0d8 commit 254cc23
Show file tree
Hide file tree
Showing 12 changed files with 378 additions and 87 deletions.
81 changes: 71 additions & 10 deletions bench/action.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ import (
"mime/multipart"
"net/http"
"net/url"
"strconv"
"strings"
"time"

"github.com/isucon/isucandar/agent"
)
Expand All @@ -31,7 +33,9 @@ func PostAdminTenantsAddAction(ctx context.Context, name, displayName string, ag
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

return ag.Do(ctx, req)
return RequestWithRetry(ctx, func() (*http.Response, error) {
return ag.Do(ctx, req)
})
}

func GetAdminTenantsBillingAction(ctx context.Context, beforeTenantID string, ag *agent.Agent) (*http.Response, error) {
Expand All @@ -44,15 +48,19 @@ func GetAdminTenantsBillingAction(ctx context.Context, beforeTenantID string, ag
return nil, err
}

return ag.Do(ctx, req)
return RequestWithRetry(ctx, func() (*http.Response, error) {
return ag.Do(ctx, req)
})
}

func GetOrganizerPlayersListAction(ctx context.Context, ag *agent.Agent) (*http.Response, error) {
req, err := ag.GET("/api/organizer/players")
if err != nil {
return nil, err
}
return ag.Do(ctx, req)
return RequestWithRetry(ctx, func() (*http.Response, error) {
return ag.Do(ctx, req)
})
}

func PostOrganizerPlayersAddAction(ctx context.Context, playerDisplayNames []string, ag *agent.Agent) (*http.Response, error) {
Expand All @@ -66,7 +74,9 @@ func PostOrganizerPlayersAddAction(ctx context.Context, playerDisplayNames []str
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

return ag.Do(ctx, req)
return RequestWithRetry(ctx, func() (*http.Response, error) {
return ag.Do(ctx, req)
})
}

func PostOrganizerApiPlayerDisqualifiedAction(ctx context.Context, playerID string, ag *agent.Agent) (*http.Response, error) {
Expand All @@ -76,7 +86,9 @@ func PostOrganizerApiPlayerDisqualifiedAction(ctx context.Context, playerID stri
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

return ag.Do(ctx, req)
return RequestWithRetry(ctx, func() (*http.Response, error) {
return ag.Do(ctx, req)
})
}

func PostOrganizerCompetitionsAddAction(ctx context.Context, title string, ag *agent.Agent) (*http.Response, error) {
Expand All @@ -88,7 +100,9 @@ func PostOrganizerCompetitionsAddAction(ctx context.Context, title string, ag *a
}

req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
return ag.Do(ctx, req)
return RequestWithRetry(ctx, func() (*http.Response, error) {
return ag.Do(ctx, req)
})
}

func PostOrganizerCompetitionFinishAction(ctx context.Context, competitionId string, ag *agent.Agent) (*http.Response, error) {
Expand All @@ -97,7 +111,9 @@ func PostOrganizerCompetitionFinishAction(ctx context.Context, competitionId str
return nil, err
}

return ag.Do(ctx, req)
return RequestWithRetry(ctx, func() (*http.Response, error) {
return ag.Do(ctx, req)
})
}

func PostOrganizerCompetitionScoreAction(ctx context.Context, competitionId string, csv []byte, ag *agent.Agent) (*http.Response, error) {
Expand All @@ -117,7 +133,9 @@ func PostOrganizerCompetitionScoreAction(ctx context.Context, competitionId stri
return nil, err
}
req.Header.Set("Content-Type", mw.FormDataContentType())
return ag.Do(ctx, req)
return RequestWithRetry(ctx, func() (*http.Response, error) {
return ag.Do(ctx, req)
})
}

func GetOrganizerBillingAction(ctx context.Context, ag *agent.Agent) (*http.Response, error) {
Expand All @@ -126,7 +144,9 @@ func GetOrganizerBillingAction(ctx context.Context, ag *agent.Agent) (*http.Resp
return nil, err
}

return ag.Do(ctx, req)
return RequestWithRetry(ctx, func() (*http.Response, error) {
return ag.Do(ctx, req)
})
}

func GetOrganizerCompetitionsAction(ctx context.Context, ag *agent.Agent) (*http.Response, error) {
Expand All @@ -135,7 +155,9 @@ func GetOrganizerCompetitionsAction(ctx context.Context, ag *agent.Agent) (*http
return nil, err
}

return ag.Do(ctx, req)
return RequestWithRetry(ctx, func() (*http.Response, error) {
return ag.Do(ctx, req)
})
}

func GetPlayerAction(ctx context.Context, playerID string, ag *agent.Agent) (*http.Response, error) {
Expand Down Expand Up @@ -168,3 +190,42 @@ func GetPlayerCompetitionsAction(ctx context.Context, ag *agent.Agent) (*http.Re

return ag.Do(ctx, req)
}

// 429 Too Many Requestsの場合にretry after分待ってretryする
func RequestWithRetry(ctx context.Context, fn func() (*http.Response, error)) (*http.Response, error) {
var res *http.Response
var err error

for {
res, err = fn()
if err != nil {
break
}

if res.StatusCode != http.StatusTooManyRequests {
break
}

ra := res.Header.Get("retry-after")

if len(ra) != 1 {
err = fmt.Errorf("invalid retry-after header")
break
}

var sec int
sec, err = strconv.Atoi(string(ra[0]))
if err != nil {
break
}

if sec < 0 {
err = fmt.Errorf("invalid retry-after header")
break
}

AdminLogger.Printf("RequestWithRetry retry: %ds %v", sec, res.Request.URL.Path)
SleepWithCtx(ctx, time.Second*time.Duration(sec))
}
return res, err
}
23 changes: 19 additions & 4 deletions bench/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,31 @@ const (
ConstNewTenantScenarioPlayerWorkerNum = 10 // 作成するplayer worker数

// PopularTenantScenario
ConstPopularTenantScenarioScoreRepeat = 2
ConstPopularTenantScenarioScoreRepeat = 2 // 一周のスコア入稿回数
ConstPopularTenantScenarioAddScoreNum = 100 // 1度のスコア入稿で増える数

// PlayerScenario
ConstPlayerScenarioCompetitionLoopCount = 10 // 一周でいくつ大会を見るか
ConstPlayerScenarioMaxPlayerCount = 10 // 大会1つあたり何人のプレイヤー詳細を見るか(最大値)

// PeacefulTenantScenario
ConstPeacefulTenantScenarioIDRange = 20 // 破壊的シナリオを許容するtenantID幅 後ろn件

// BillingValidateScenario
ConstBillingValidateScenarioPlayerNum = 100
// TenantBillingValidateScenario
ConstTenantBillingValidateScenarioPlayerNum = 100 // TenantBilling検証用テナントのplayer数

// AdminBillingValidateScenario
ConstAdminBillingValidateScenarioPlayerNum = 100 // TenantBilling検証用テナントのplayer数
)

var (
// NOTE: 初期データ範囲について(0-based)
// 0: 巨大テナント
// 1 ~ 39: 人気テナント
// 40 ~ 69: AdminBilling検証用テナント
// 70 ~ 99: 破壊的操作テナント
// 100: prepare validate tenant

ConstPopularTenantScenarioIDRange = []int{0, 39} // 利用する初期データのテナントID幅
ConstAdminBillingValidateScenarioIDRange = []int{40, 69} // 利用する初期データのテナントID幅
ConstPeacefulTenantScenarioIDRange = []int{70, 99} // 利用する初期データのテナントID幅
)
1 change: 1 addition & 0 deletions bench/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ require (
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/goccy/go-json v0.9.7 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/google/go-cmp v0.5.8 // indirect
github.com/jaswdr/faker v1.10.2 // indirect
github.com/jmoiron/sqlx v1.3.5 // indirect
github.com/labstack/echo/v4 v4.7.2 // indirect
Expand Down
2 changes: 2 additions & 0 deletions bench/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM=
github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/isucon/isucandar v0.0.0-20220322062028-6dd56dc57d72 h1:mCWYjY0ZaMGAFqwCC/hsEZZl+R8mymuVIRhmVO8r/EE=
github.com/isucon/isucandar v0.0.0-20220322062028-6dd56dc57d72/go.mod h1:1j6H6zxOUW/sSAOGay1iE0CW6mM58U/OTIKvKXwbRaQ=
github.com/jaswdr/faker v1.10.2 h1:GK03wuDqa8V6BE+2VRr3DJ/G4T0iUDCzVoBCj5TM4b8=
Expand Down
15 changes: 14 additions & 1 deletion bench/scenario.go
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,16 @@ func (sc *Scenario) Load(c context.Context, step *isucandar.BenchmarkStep) error

// Tenant Billingの整合性をチェックするシナリオ
{
wkr, err := sc.BillingValidateWorker(step, 1)
wkr, err := sc.TenantBillingValidateWorker(step, 1)
if err != nil {
return err
}
sc.WorkerCh <- wkr
}

// Admin Billingの整合性をチェックするシナリオ
{
wkr, err := sc.AdminBillingValidateWorker(step, 1)
if err != nil {
return err
}
Expand All @@ -234,6 +243,10 @@ func (sc *Scenario) Load(c context.Context, step *isucandar.BenchmarkStep) error
case <-ctx.Done():
end = true
case w := <-sc.WorkerCh: // workerを起動する
// debug: 一つのworkerのみを立ち上げる
// if w.String() != "AdminBillingValidateWorker" {
// continue
// }
wg.Add(1)
sc.CountWorker(w.String())
go func(w Worker) {
Expand Down
141 changes: 141 additions & 0 deletions bench/scenario_admin_billing_validate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
package bench

import (
"context"
"fmt"
"time"

"github.com/isucon/isucandar"
"github.com/isucon/isucandar/worker"
isuports "github.com/isucon/isucon12-qualify/webapp/go"
)

type adminBillingValidateWorker struct {
worker *worker.Worker
}

func (adminBillingValidateWorker) String() string {
return "AdminBillingValidateWorker"
}
func (w *adminBillingValidateWorker) Process(ctx context.Context) { w.worker.Process(ctx) }

func (sc *Scenario) AdminBillingValidateWorker(step *isucandar.BenchmarkStep, p int32) (*adminBillingValidateWorker, error) {
scTag := ScenarioTagAdminBillingValidate
w, err := worker.NewWorker(func(ctx context.Context, _ int) {
if err := sc.AdminBillingValidate(ctx, step); err != nil {
sc.ScenarioError(scTag, err)
SleepWithCtx(ctx, SleepOnError)
}
},
worker.WithInfinityLoop(),
worker.WithUnlimitedParallelism(),
)
if err != nil {
return nil, err
}
w.SetParallelism(p)

return &adminBillingValidateWorker{
worker: w,
}, nil
}

func (sc *Scenario) AdminBillingValidate(ctx context.Context, step *isucandar.BenchmarkStep) error {
report := timeReporter("SaaS管理者請求検証シナリオ")
defer report()
scTag := ScenarioTagAdminBillingValidate
sc.ScenarioStart(scTag)

_, adminAg, err := sc.GetAccountAndAgent(AccountRoleAdmin, "admin", "admin")
if err != nil {
return err
}

// 初期データからテナント選ぶ
index := randomRange(ConstAdminBillingValidateScenarioIDRange)
tenant := sc.InitialDataTenant[int64(index)]

// indexが含まれる区間がとれるAdminBillingのbefore
var billingBeforeTenantID string
{
rangeEnd := ConstAdminBillingValidateScenarioIDRange[1]
n := index + 10
if rangeEnd < n {
n = rangeEnd
}
billingBeforeTenantID = fmt.Sprintf("%d", sc.InitialDataTenant[int64(n)].TenantID)
}

// 最初の状態のBilling
var billingResultTenants []isuports.TenantWithBilling
{
res, err := GetAdminTenantsBillingAction(ctx, billingBeforeTenantID, adminAg)
v := ValidateResponse("テナント別の請求ダッシュボード", step, res, err, WithStatusCode(200),
WithSuccessResponse(func(r ResponseAPITenantsBilling) error {
billingResultTenants = r.Data.Tenants
return nil
}),
)
if v.IsEmpty() {
sc.AddScoreByScenario(step, ScoreGETAdminTenantsBilling, scTag)
} else {
sc.AddErrorCount()
return v
}
}

// 大会を開催、Billing確定まで進める
_, orgAg, err := sc.GetAccountAndAgent(AccountRoleOrganizer, tenant.TenantName, "organizer")
if err != nil {
return err
}

conf := &OrganizerJobConfig{
orgAg: orgAg,
scTag: scTag,
tenantName: tenant.TenantName,
scoreRepeat: 1,
scoreInterval: 0,
addScoreNum: 0,
}
if err := sc.OrganizerJob(ctx, step, conf); err != nil {
return err
}

// 反映まで3秒まで猶予がある
SleepWithCtx(ctx, time.Second*3)

// 反映確認

// チェック項目
// 合計金額が増えていること
// TODO: 必要に応じて追加, ただしOrganizerJobによって増えた金額は現状取れない
sumYen := int64(0)
for _, t := range billingResultTenants {
sumYen += t.BillingYen
}

{
res, err := GetAdminTenantsBillingAction(ctx, billingBeforeTenantID, adminAg)
v := ValidateResponse("テナント別の請求ダッシュボード", step, res, err, WithStatusCode(200),
WithSuccessResponse(func(r ResponseAPITenantsBilling) error {
resultYen := int64(0)
for _, t := range r.Data.Tenants {
resultYen += t.BillingYen
}
if resultYen <= sumYen {
return fmt.Errorf("全テナントの合計金額が正しくありません 金額は増えている必要があります (want: >%d, got:%d)", sumYen, resultYen)
}
return nil
}),
)
if v.IsEmpty() {
sc.AddScoreByScenario(step, ScoreGETAdminTenantsBilling, scTag)
} else {
sc.AddErrorCount()
return v
}
}

return nil
}
Loading

0 comments on commit 254cc23

Please sign in to comment.