From d6dce31adc63882ff051913e661eaeb421875138 Mon Sep 17 00:00:00 2001 From: ToshihitoKon Date: Tue, 5 Jul 2022 11:04:56 +0900 Subject: [PATCH 01/10] empty commit From 64c3710080b8d853a308b57be32e44fbb2fbe2e7 Mon Sep 17 00:00:00 2001 From: ToshihitoKon Date: Tue, 5 Jul 2022 16:08:08 +0900 Subject: [PATCH 02/10] =?UTF-8?q?retry=20after=E3=81=A0=E3=81=91retry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bench/action.go | 76 +++++++++++++++++++++++++++++++----- bench/scenario_validation.go | 1 - bench/validation.go | 31 +-------------- 3 files changed, 67 insertions(+), 41 deletions(-) diff --git a/bench/action.go b/bench/action.go index f2413d08..eb545011 100644 --- a/bench/action.go +++ b/bench/action.go @@ -7,7 +7,9 @@ import ( "mime/multipart" "net/http" "net/url" + "strconv" "strings" + "time" "github.com/isucon/isucandar/agent" ) @@ -31,7 +33,9 @@ func PostAdminTenantsAddAction(ctx context.Context, name, displayName string, ag } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - return ag.Do(ctx, req) + return RequestWithRetry(ctx, func() (*http.Response, error) { + return ag.Do(ctx, req) + }) } func GetAdminTenantsBillingAction(ctx context.Context, beforeTenantID string, ag *agent.Agent) (*http.Response, error) { @@ -44,7 +48,9 @@ func GetAdminTenantsBillingAction(ctx context.Context, beforeTenantID string, ag return nil, err } - return ag.Do(ctx, req) + return RequestWithRetry(ctx, func() (*http.Response, error) { + return ag.Do(ctx, req) + }) } func GetOrganizerPlayersListAction(ctx context.Context, ag *agent.Agent) (*http.Response, error) { @@ -52,7 +58,9 @@ func GetOrganizerPlayersListAction(ctx context.Context, ag *agent.Agent) (*http. if err != nil { return nil, err } - return ag.Do(ctx, req) + return RequestWithRetry(ctx, func() (*http.Response, error) { + return ag.Do(ctx, req) + }) } func PostOrganizerPlayersAddAction(ctx context.Context, playerDisplayNames []string, ag *agent.Agent) (*http.Response, error) { @@ -66,7 +74,9 @@ func PostOrganizerPlayersAddAction(ctx context.Context, playerDisplayNames []str } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - return ag.Do(ctx, req) + return RequestWithRetry(ctx, func() (*http.Response, error) { + return ag.Do(ctx, req) + }) } func PostOrganizerApiPlayerDisqualifiedAction(ctx context.Context, playerID string, ag *agent.Agent) (*http.Response, error) { @@ -76,7 +86,9 @@ func PostOrganizerApiPlayerDisqualifiedAction(ctx context.Context, playerID stri } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - return ag.Do(ctx, req) + return RequestWithRetry(ctx, func() (*http.Response, error) { + return ag.Do(ctx, req) + }) } func PostOrganizerCompetitionsAddAction(ctx context.Context, title string, ag *agent.Agent) (*http.Response, error) { @@ -88,7 +100,9 @@ func PostOrganizerCompetitionsAddAction(ctx context.Context, title string, ag *a } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - return ag.Do(ctx, req) + return RequestWithRetry(ctx, func() (*http.Response, error) { + return ag.Do(ctx, req) + }) } func PostOrganizerCompetitionFinishAction(ctx context.Context, competitionId string, ag *agent.Agent) (*http.Response, error) { @@ -97,7 +111,9 @@ func PostOrganizerCompetitionFinishAction(ctx context.Context, competitionId str return nil, err } - return ag.Do(ctx, req) + return RequestWithRetry(ctx, func() (*http.Response, error) { + return ag.Do(ctx, req) + }) } func PostOrganizerCompetitionScoreAction(ctx context.Context, competitionId string, csv []byte, ag *agent.Agent) (*http.Response, error) { @@ -117,7 +133,9 @@ func PostOrganizerCompetitionScoreAction(ctx context.Context, competitionId stri return nil, err } req.Header.Set("Content-Type", mw.FormDataContentType()) - return ag.Do(ctx, req) + return RequestWithRetry(ctx, func() (*http.Response, error) { + return ag.Do(ctx, req) + }) } func GetOrganizerBillingAction(ctx context.Context, ag *agent.Agent) (*http.Response, error) { @@ -126,7 +144,9 @@ func GetOrganizerBillingAction(ctx context.Context, ag *agent.Agent) (*http.Resp return nil, err } - return ag.Do(ctx, req) + return RequestWithRetry(ctx, func() (*http.Response, error) { + return ag.Do(ctx, req) + }) } func GetOrganizerCompetitionsAction(ctx context.Context, ag *agent.Agent) (*http.Response, error) { @@ -135,7 +155,9 @@ func GetOrganizerCompetitionsAction(ctx context.Context, ag *agent.Agent) (*http return nil, err } - return ag.Do(ctx, req) + return RequestWithRetry(ctx, func() (*http.Response, error) { + return ag.Do(ctx, req) + }) } func GetPlayerAction(ctx context.Context, playerID string, ag *agent.Agent) (*http.Response, error) { @@ -168,3 +190,37 @@ func GetPlayerCompetitionsAction(ctx context.Context, ag *agent.Agent) (*http.Re return ag.Do(ctx, req) } + +// 429 Too Many Requestsの場合にretry after分待ってretryする +func RequestWithRetry(ctx context.Context, fn func() (*http.Response, error)) (*http.Response, error) { + var res *http.Response + var err error + + for { + res, err = fn() + if err != nil { + break + } + + if res.StatusCode != http.StatusTooManyRequests { + break + } + + AdminLogger.Printf("RequestWithRetry retry: %v", res.Request.URL.Path) + ra := res.Header.Get("retry-after") + + if len(ra) != 1 { + err = fmt.Errorf("invalid retry-after header") + break + } + + var sec int + sec, err = strconv.Atoi(string(ra[0])) + if err != nil { + break + } + + SleepWithCtx(ctx, time.Second*time.Duration(sec)) + } + return res, err +} diff --git a/bench/scenario_validation.go b/bench/scenario_validation.go index 217147e6..825d87f5 100644 --- a/bench/scenario_validation.go +++ b/bench/scenario_validation.go @@ -58,7 +58,6 @@ func (sc *Scenario) ValidationScenario(ctx context.Context, step *isucandar.Benc if !v.IsEmpty() { return v } - // テナント追加 不正リクエストチェック invalidNames := map[string]int{ "valid-tenantid": http.StatusBadRequest, // 重複するname diff --git a/bench/validation.go b/bench/validation.go index 0efb1dcb..ad6b7040 100644 --- a/bench/validation.go +++ b/bench/validation.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "net/http" - "strconv" "strings" "github.com/isucon/isucandar" @@ -28,9 +27,6 @@ type ValidationError struct { Errors []error Title string Canceled bool - - isTooManyRequest bool - retryAfter int } // error インターフェースを満たす Error メソッド @@ -80,9 +76,7 @@ func ReadResponse(res *http.Response) *Response { // 複数のバリデータ関数を受け取ってすべてでレスポンスを検証し、 ValidationError を返す func ValidateResponse(title string, step *isucandar.BenchmarkStep, res *http.Response, err error, validators ...ResponseValidator) ValidationError { ve := ValidationError{ - Title: title, - isTooManyRequest: false, - retryAfter: 0, + Title: title, } defer func() { ve.Add(step) @@ -112,35 +106,12 @@ func ValidateResponse(title string, step *isucandar.BenchmarkStep, res *http.Res } } - // 429 Too Many Requestsならretryするために保存しておく - if response.Response.StatusCode == http.StatusTooManyRequests { - ve.isTooManyRequest = true - ra := response.Response.Header.Get("retry-after") - if len(ra) == 1 { - sec, err := strconv.Atoi(string(ra[0])) - if err != nil { - ve.Errors = append(ve.Errors, failure.NewError(ErrValidation, err)) - } else { - ve.retryAfter = sec - } - } - } - if !ve.IsEmpty() { ContestantLogger.Print(ve.Error()) } return ve } -// 429 too many requestから返ってきたValidationErrorなら詳細を返す -// 使用例: -// if ok, ra := v.IsTooManyRequest; ok { -// sc.SleepWithCtx(ctx, ra*time.Second) -// } -func (ve ValidationError) IsTooManyRequest(ctx context.Context) (bool, int) { - return ve.isTooManyRequest, ve.retryAfter -} - func WithCacheControlPrivate() ResponseValidator { return func(r *Response) error { if !strings.Contains(r.Response.Header.Get("Cache-Control"), "private") { From 198156594aae03d836d5e6b52867f07c6bc25cbe Mon Sep 17 00:00:00 2001 From: ToshihitoKon Date: Tue, 5 Jul 2022 16:29:26 +0900 Subject: [PATCH 03/10] retry after check --- bench/action.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bench/action.go b/bench/action.go index eb545011..a77f3df0 100644 --- a/bench/action.go +++ b/bench/action.go @@ -206,7 +206,6 @@ func RequestWithRetry(ctx context.Context, fn func() (*http.Response, error)) (* break } - AdminLogger.Printf("RequestWithRetry retry: %v", res.Request.URL.Path) ra := res.Header.Get("retry-after") if len(ra) != 1 { @@ -220,6 +219,12 @@ func RequestWithRetry(ctx context.Context, fn func() (*http.Response, error)) (* break } + if sec < 0 { + err = fmt.Errorf("invalid retry-after header") + break + } + + AdminLogger.Printf("RequestWithRetry retry: %ds %v", sec, res.Request.URL.Path) SleepWithCtx(ctx, time.Second*time.Duration(sec)) } return res, err From 17850154d5b9947503e4b806990cdfd0c72e5dc8 Mon Sep 17 00:00:00 2001 From: ToshihitoKon Date: Tue, 5 Jul 2022 16:29:51 +0900 Subject: [PATCH 04/10] =?UTF-8?q?debug:=2050%=E3=81=AE=E7=A2=BA=E7=8E=87?= =?UTF-8?q?=E3=81=A7429=E3=82=92=E8=BF=94=E3=81=99player?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- webapp/go/isuports.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/webapp/go/isuports.go b/webapp/go/isuports.go index 8d4e97ad..6d7804cf 100644 --- a/webapp/go/isuports.go +++ b/webapp/go/isuports.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "math/rand" "net/http" "os" "os/exec" @@ -720,6 +721,10 @@ type PlayersListHandlerResult struct { // GET /api/organizer/players // 参加者一覧を返す func playersListHandler(c echo.Context) error { + if rand.Intn(2) == 0 { // DEBUG: 50%の確率で429を返す + c.Response().Header().Set("Retry-After", "1") + return echo.NewHTTPError(http.StatusTooManyRequests, "debug too many requests") + } ctx := context.Background() v, err := parseViewer(c) if err != nil { From 93103defde150ebfafcc05c287632773166d4e53 Mon Sep 17 00:00:00 2001 From: ToshihitoKon Date: Tue, 5 Jul 2022 16:30:03 +0900 Subject: [PATCH 05/10] =?UTF-8?q?Revert=20"debug:=2050%=E3=81=AE=E7=A2=BA?= =?UTF-8?q?=E7=8E=87=E3=81=A7429=E3=82=92=E8=BF=94=E3=81=99player"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 17850154d5b9947503e4b806990cdfd0c72e5dc8. --- webapp/go/isuports.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/webapp/go/isuports.go b/webapp/go/isuports.go index 6d7804cf..8d4e97ad 100644 --- a/webapp/go/isuports.go +++ b/webapp/go/isuports.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - "math/rand" "net/http" "os" "os/exec" @@ -721,10 +720,6 @@ type PlayersListHandlerResult struct { // GET /api/organizer/players // 参加者一覧を返す func playersListHandler(c echo.Context) error { - if rand.Intn(2) == 0 { // DEBUG: 50%の確率で429を返す - c.Response().Header().Set("Retry-After", "1") - return echo.NewHTTPError(http.StatusTooManyRequests, "debug too many requests") - } ctx := context.Background() v, err := parseViewer(c) if err != nil { From a25c509650e951d80ed4488f343a8393bf4552aa Mon Sep 17 00:00:00 2001 From: ToshihitoKon Date: Tue, 5 Jul 2022 17:14:47 +0900 Subject: [PATCH 06/10] =?UTF-8?q?ranking=E3=81=8C=E6=B6=88=E3=81=88?= =?UTF-8?q?=E3=82=8B=E3=81=93=E3=81=A8=E3=82=92=E7=A2=BA=E8=AA=8D=E3=81=99?= =?UTF-8?q?=E3=82=8B=E6=95=B4=E5=90=88=E6=80=A7=E3=83=81=E3=82=A7=E3=83=83?= =?UTF-8?q?=E3=82=AF=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bench/scenario_validation.go | 39 ++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/bench/scenario_validation.go b/bench/scenario_validation.go index 825d87f5..aff8ca28 100644 --- a/bench/scenario_validation.go +++ b/bench/scenario_validation.go @@ -660,10 +660,37 @@ func (sc *Scenario) ValidationScenario(ctx context.Context, step *isucandar.Benc return &v } } - // 終了する + + // 結果を引く + { + res, err := GetPlayerCompetitionRankingAction(ctx, rankingCheckCompetitionID, "", playerAg) + v := ValidateResponse("大会内のランキング取得: ページングなし,上限100件", step, res, err, WithStatusCode(200), + WithSuccessResponse(func(r ResponseAPICompetitionRanking) error { + if 100 != len(r.Data.Ranks) { + return fmt.Errorf("大会のランキングの結果の最大は100件である必要があります (want: %d, got: %d)", 100, len(r.Data.Ranks)) + } + return nil + }), + ) + if !v.IsEmpty() { + return &v + } + } + + // 最後に入稿されたCSVのみが有効なことを確認 + lastCheckScore := ScoreRows{} { - res, err := PostOrganizerCompetitionFinishAction(ctx, rankingCheckCompetitionID, orgAg) - v := ValidateResponse("大会終了", step, res, err, WithStatusCode(200)) + csv := lastCheckScore.CSV() + res, err := PostOrganizerCompetitionScoreAction(ctx, rankingCheckCompetitionID, []byte(csv), orgAg) + v := ValidateResponse("大会結果CSV入稿: 空", step, res, err, + WithStatusCode(200), + WithSuccessResponse(func(r ResponseAPICompetitionResult) error { + if r.Data.Rows != int64(len(lastCheckScore)) { + return fmt.Errorf("大会結果CSV入稿レスポンスのRowsが異なります (want: %d, got: %d)", len(lastCheckScore), r.Data.Rows) + } + return nil + }), + ) if !v.IsEmpty() { return &v } @@ -672,10 +699,10 @@ func (sc *Scenario) ValidationScenario(ctx context.Context, step *isucandar.Benc // 結果を引く { res, err := GetPlayerCompetitionRankingAction(ctx, rankingCheckCompetitionID, "", playerAg) - v := ValidateResponse("大会内のランキング取得: ページングなし", step, res, err, WithStatusCode(200), + v := ValidateResponse("大会内のランキング取得: 結果が空", step, res, err, WithStatusCode(200), WithSuccessResponse(func(r ResponseAPICompetitionRanking) error { - if 100 != len(r.Data.Ranks) { - return fmt.Errorf("大会のランキングの結果の最大は100件である必要があります (want: %d, got: %d)", 100, len(r.Data.Ranks)) + if len(lastCheckScore) != len(r.Data.Ranks) { + return fmt.Errorf("大会のランキングの結果が違います (want: %d, got: %d)", len(lastCheckScore), len(r.Data.Ranks)) } return nil }), From 0e3af6d9393a8516823899fe61e10d5b2c3bcf55 Mon Sep 17 00:00:00 2001 From: ToshihitoKon Date: Tue, 5 Jul 2022 19:17:17 +0900 Subject: [PATCH 07/10] =?UTF-8?q?player=20handler=E3=81=AEcache=E3=83=81?= =?UTF-8?q?=E3=82=A7=E3=83=83=E3=82=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bench/scenario.go | 4 ++ bench/scenario_billing_validate.go | 1 + bench/scenario_peaceful_tenant.go | 77 ++++++++++++++++++++++++++---- bench/scenario_validation.go | 3 +- 4 files changed, 74 insertions(+), 11 deletions(-) diff --git a/bench/scenario.go b/bench/scenario.go index ee774605..4e488d75 100644 --- a/bench/scenario.go +++ b/bench/scenario.go @@ -234,6 +234,10 @@ func (sc *Scenario) Load(c context.Context, step *isucandar.BenchmarkStep) error case <-ctx.Done(): end = true case w := <-sc.WorkerCh: // workerを起動する + // debug: 一つのworkerのみを立ち上げる + // if w.String() != "PeacefulTenantScenarioWorker" { + // continue + // } wg.Add(1) sc.CountWorker(w.String()) go func(w Worker) { diff --git a/bench/scenario_billing_validate.go b/bench/scenario_billing_validate.go index 8655b97f..f90ff526 100644 --- a/bench/scenario_billing_validate.go +++ b/bench/scenario_billing_validate.go @@ -41,6 +41,7 @@ func (sc *Scenario) BillingValidateWorker(step *isucandar.BenchmarkStep, p int32 }, nil } +// TODO: 1テナントで複数大会作成する func (sc *Scenario) BillingValidate(ctx context.Context, step *isucandar.BenchmarkStep) error { report := timeReporter("テナント請求検証シナリオ") defer report() diff --git a/bench/scenario_peaceful_tenant.go b/bench/scenario_peaceful_tenant.go index a2eb6672..ab171882 100644 --- a/bench/scenario_peaceful_tenant.go +++ b/bench/scenario_peaceful_tenant.go @@ -2,6 +2,7 @@ package bench import ( "context" + "fmt" "math/rand" "time" @@ -62,7 +63,10 @@ func (sc *Scenario) PeacefulTenantScenario(ctx context.Context, step *isucandar. v := ValidateResponse("テナントのプレイヤー一覧取得", step, res, err, WithStatusCode(200), WithSuccessResponse(func(r ResponseAPIPlayersList) error { for _, player := range r.Data.Players { - playerIDs = append(playerIDs, player.ID) + // 失格じゃないプレイヤーを列挙する + if !player.IsDisqualified { + playerIDs = append(playerIDs, player.ID) + } } return nil }), @@ -74,11 +78,45 @@ func (sc *Scenario) PeacefulTenantScenario(ctx context.Context, step *isucandar. return v } } - playerID := playerIDs[rand.Intn(len(playerIDs))] + n = rand.Intn(len(playerIDs) - 1) + disqualifyPlayerID := playerIDs[n] + checkerPlayerID := playerIDs[n+1] + + _, disqualifiedPlayerAg, err := sc.GetAccountAndAgent(AccountRolePlayer, tenant.TenantName, disqualifyPlayerID) + if err != nil { + return err + } + _, checkerPlayerAg, err := sc.GetAccountAndAgent(AccountRolePlayer, tenant.TenantName, checkerPlayerID) + if err != nil { + return err + } + + // 失格前に失格にするプレイヤーを見に行く + { + res, err := GetPlayerAction(ctx, disqualifyPlayerID, checkerPlayerAg) + v := ValidateResponse("プレイヤーと戦績情報取得: 失格前", step, res, err, WithStatusCode(200), + WithSuccessResponse(func(r ResponseAPIPlayer) error { + if disqualifyPlayerID != r.Data.Player.ID { + return fmt.Errorf("参照したプレイヤー名が違います (want: %s, got: %s)", disqualifyPlayerID, r.Data.Player.ID) + } + if false != r.Data.Player.IsDisqualified { + return fmt.Errorf("失格状態が違います (want: %v, got: %v)", false, r.Data.Player.IsDisqualified) + } + + return nil + }), + ) + if v.IsEmpty() { + sc.AddScoreByScenario(step, ScoreGETPlayerDetails, scTag) + } else { + sc.AddErrorCount() + return v + } + } // プレイヤーを1人失格にする { - res, err := PostOrganizerApiPlayerDisqualifiedAction(ctx, playerID, orgAg) + res, err := PostOrganizerApiPlayerDisqualifiedAction(ctx, disqualifyPlayerID, orgAg) v := ValidateResponse("プレイヤーを失格にする", step, res, err, WithStatusCode(200), WithSuccessResponse(func(r ResponseAPIPlayerDisqualified) error { _ = r @@ -93,14 +131,10 @@ func (sc *Scenario) PeacefulTenantScenario(ctx context.Context, step *isucandar. } } - _, playerAg, err := sc.GetAccountAndAgent(AccountRoleOrganizer, tenant.TenantName, playerID) - if err != nil { - return err - } - + // 失格プレイヤーで情報を見に行く 403 { - res, err := GetPlayerCompetitionsAction(ctx, playerAg) - v := ValidateResponse("テナント内の大会情報取得", step, res, err, WithStatusCode(403)) + res, err := GetPlayerCompetitionsAction(ctx, disqualifiedPlayerAg) + v := ValidateResponse("テナント内の大会情報取得: 失格済みプレイヤーは403で弾く", step, res, err, WithStatusCode(403)) if v.IsEmpty() { sc.AddScoreByScenario(step, ScoreGETPlayerCompetitions, scTag) } else { @@ -109,6 +143,29 @@ func (sc *Scenario) PeacefulTenantScenario(ctx context.Context, step *isucandar. } } + // 失格プレイヤーを見に行く IsDisqualifiedが更新されていることをチェック + { + res, err := GetPlayerAction(ctx, disqualifyPlayerID, checkerPlayerAg) + v := ValidateResponse("プレイヤーと戦績情報取得: 失格済みプレイヤーを見に行く", step, res, err, WithStatusCode(200), + WithSuccessResponse(func(r ResponseAPIPlayer) error { + if disqualifyPlayerID != r.Data.Player.ID { + return fmt.Errorf("参照したプレイヤー名が違います (want: %s, got: %s)", disqualifyPlayerID, r.Data.Player.ID) + } + if true != r.Data.Player.IsDisqualified { + return fmt.Errorf("失格状態が違います (want: %v, got: %v)", true, r.Data.Player.IsDisqualified) + } + + return nil + }), + ) + if v.IsEmpty() { + sc.AddScoreByScenario(step, ScoreGETPlayerDetails, scTag) + } else { + sc.AddCriticalCount() // 反映されていないのはCritical + return v + } + } + // sleep 1.0s ~ 2.0s sleepms := 1000 + rand.Intn(1000) SleepWithCtx(ctx, time.Millisecond*time.Duration(sleepms)) diff --git a/bench/scenario_validation.go b/bench/scenario_validation.go index aff8ca28..c8c3e6e4 100644 --- a/bench/scenario_validation.go +++ b/bench/scenario_validation.go @@ -16,7 +16,7 @@ var ( notExistName = "null-null" // 存在しない想定のName(tenant用) ) -// ベンチ実行後の整合性検証シナリオ +// ベンチ実行前の整合性検証シナリオ // isucandar.ValidateScenarioを満たすメソッド // isucandar.Benchmark の validation ステップで実行される func (sc *Scenario) ValidationScenario(ctx context.Context, step *isucandar.BenchmarkStep) error { @@ -169,6 +169,7 @@ func (sc *Scenario) ValidationScenario(ctx context.Context, step *isucandar.Benc } // NOTE: 不正リクエストチェックなし } + { idx := disqualifiedPlayerIndex res, err := PostOrganizerApiPlayerDisqualifiedAction(ctx, playerIDs[idx], orgAg) From f58b41041ae8e6a0b1213d70b53b7f534b4c41dc Mon Sep 17 00:00:00 2001 From: ToshihitoKon Date: Tue, 5 Jul 2022 20:47:40 +0900 Subject: [PATCH 08/10] =?UTF-8?q?AdminBilling=E3=82=92=E6=A4=9C=E8=A8=BC?= =?UTF-8?q?=E3=81=99=E3=82=8B=E3=82=B7=E3=83=8A=E3=83=AA=E3=82=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bench/constants.go | 23 +++- bench/go.mod | 1 + bench/go.sum | 2 + bench/scenario.go | 13 +- bench/scenario_admin_billing_validate.go | 117 ++++++++++++++++++ bench/scenario_peaceful_tenant.go | 6 +- ...go => scenario_tenant_billing_validate.go} | 32 ++--- bench/scenario_util.go | 7 ++ bench/tags.go | 6 +- 9 files changed, 176 insertions(+), 31 deletions(-) create mode 100644 bench/scenario_admin_billing_validate.go rename bench/{scenario_billing_validate.go => scenario_tenant_billing_validate.go} (89%) diff --git a/bench/constants.go b/bench/constants.go index 0322ca2e..9035480b 100644 --- a/bench/constants.go +++ b/bench/constants.go @@ -10,7 +10,7 @@ const ( ConstNewTenantScenarioPlayerWorkerNum = 10 // 作成するplayer worker数 // PopularTenantScenario - ConstPopularTenantScenarioScoreRepeat = 2 + ConstPopularTenantScenarioScoreRepeat = 2 // 一周のスコア入稿回数 ConstPopularTenantScenarioAddScoreNum = 100 // 1度のスコア入稿で増える数 // PlayerScenario @@ -18,8 +18,23 @@ const ( ConstPlayerScenarioMaxPlayerCount = 10 // 大会1つあたり何人のプレイヤー詳細を見るか(最大値) // PeacefulTenantScenario - ConstPeacefulTenantScenarioIDRange = 20 // 破壊的シナリオを許容するtenantID幅 後ろn件 - // BillingValidateScenario - ConstBillingValidateScenarioPlayerNum = 100 + // TenantBillingValidateScenario + ConstTenantBillingValidateScenarioPlayerNum = 100 // TenantBilling検証用テナントのplayer数 + + // AdminBillingValidateScenario + ConstAdminBillingValidateScenarioPlayerNum = 100 // TenantBilling検証用テナントのplayer数 +) + +var ( + // NOTE: 初期データ範囲について(0-based) + // 0: 巨大テナント + // 1 ~ 39: 人気テナント + // 40 ~ 69: AdminBilling検証用テナント + // 70 ~ 99: 破壊的操作テナント + // 100: prepare validate tenant + + ConstPopularTenantScenarioIDRange = []int{0, 39} // 利用する初期データのテナントID幅 + ConstAdminBillingValidateScenarioIDRange = []int{40, 69} // 利用する初期データのテナントID幅 + ConstPeacefulTenantScenarioIDRange = []int{70, 99} // 利用する初期データのテナントID幅 ) diff --git a/bench/go.mod b/bench/go.mod index 7af56bd6..010cb852 100644 --- a/bench/go.mod +++ b/bench/go.mod @@ -16,6 +16,7 @@ require ( github.com/go-sql-driver/mysql v1.6.0 // indirect github.com/goccy/go-json v0.9.7 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/google/go-cmp v0.5.8 // indirect github.com/jaswdr/faker v1.10.2 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect github.com/labstack/echo/v4 v4.7.2 // indirect diff --git a/bench/go.sum b/bench/go.sum index 51e98305..0c704b4e 100644 --- a/bench/go.sum +++ b/bench/go.sum @@ -14,6 +14,8 @@ github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/isucon/isucandar v0.0.0-20220322062028-6dd56dc57d72 h1:mCWYjY0ZaMGAFqwCC/hsEZZl+R8mymuVIRhmVO8r/EE= github.com/isucon/isucandar v0.0.0-20220322062028-6dd56dc57d72/go.mod h1:1j6H6zxOUW/sSAOGay1iE0CW6mM58U/OTIKvKXwbRaQ= github.com/jaswdr/faker v1.10.2 h1:GK03wuDqa8V6BE+2VRr3DJ/G4T0iUDCzVoBCj5TM4b8= diff --git a/bench/scenario.go b/bench/scenario.go index 4e488d75..a4de704a 100644 --- a/bench/scenario.go +++ b/bench/scenario.go @@ -218,7 +218,16 @@ func (sc *Scenario) Load(c context.Context, step *isucandar.BenchmarkStep) error // Tenant Billingの整合性をチェックするシナリオ { - wkr, err := sc.BillingValidateWorker(step, 1) + wkr, err := sc.TenantBillingValidateWorker(step, 1) + if err != nil { + return err + } + sc.WorkerCh <- wkr + } + + // Admin Billingの整合性をチェックするシナリオ + { + wkr, err := sc.AdminBillingValidateWorker(step, 1) if err != nil { return err } @@ -235,7 +244,7 @@ func (sc *Scenario) Load(c context.Context, step *isucandar.BenchmarkStep) error end = true case w := <-sc.WorkerCh: // workerを起動する // debug: 一つのworkerのみを立ち上げる - // if w.String() != "PeacefulTenantScenarioWorker" { + // if w.String() != "AdminBillingValidateWorker" { // continue // } wg.Add(1) diff --git a/bench/scenario_admin_billing_validate.go b/bench/scenario_admin_billing_validate.go new file mode 100644 index 00000000..0e0cbe47 --- /dev/null +++ b/bench/scenario_admin_billing_validate.go @@ -0,0 +1,117 @@ +package bench + +import ( + "context" + "fmt" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/isucon/isucandar" + "github.com/isucon/isucandar/worker" + isuports "github.com/isucon/isucon12-qualify/webapp/go" +) + +type adminBillingValidateWorker struct { + worker *worker.Worker +} + +func (adminBillingValidateWorker) String() string { + return "AdminBillingValidateWorker" +} +func (w *adminBillingValidateWorker) Process(ctx context.Context) { w.worker.Process(ctx) } + +func (sc *Scenario) AdminBillingValidateWorker(step *isucandar.BenchmarkStep, p int32) (*adminBillingValidateWorker, error) { + scTag := ScenarioTagAdminBillingValidate + w, err := worker.NewWorker(func(ctx context.Context, _ int) { + if err := sc.AdminBillingValidate(ctx, step); err != nil { + sc.ScenarioError(scTag, err) + SleepWithCtx(ctx, SleepOnError) + } + }, + worker.WithInfinityLoop(), + worker.WithUnlimitedParallelism(), + ) + if err != nil { + return nil, err + } + w.SetParallelism(p) + + return &adminBillingValidateWorker{ + worker: w, + }, nil +} + +func (sc *Scenario) AdminBillingValidate(ctx context.Context, step *isucandar.BenchmarkStep) error { + report := timeReporter("SaaS管理者請求検証シナリオ") + defer report() + scTag := ScenarioTagAdminBillingValidate + sc.ScenarioStart(scTag) + + _, adminAg, err := sc.GetAccountAndAgent(AccountRoleAdmin, "admin", "admin") + if err != nil { + return err + } + + // 初期データからテナント選ぶ + index := randomRange(ConstAdminBillingValidateScenarioIDRange) + // tenant := sc.InitialDataTenant[int64(index)] + + // indexが含まれる区間がとれるAdminBillingのbefore + var billingBeforeTenantID string + { + rangeEnd := ConstAdminBillingValidateScenarioIDRange[1] + n := index + 10 + if rangeEnd < n { + n = rangeEnd + } + billingBeforeTenantID = fmt.Sprintf("%d", sc.InitialDataTenant[int64(n)].TenantID) + } + + // 最初の状態のBilling + var billingResultTenants []isuports.TenantWithBilling + { + res, err := GetAdminTenantsBillingAction(ctx, billingBeforeTenantID, adminAg) + v := ValidateResponse("テナント別の請求ダッシュボード", step, res, err, WithStatusCode(200), + WithSuccessResponse(func(r ResponseAPITenantsBilling) error { + billingResultTenants = r.Data.Tenants + return nil + }), + ) + if v.IsEmpty() { + sc.AddScoreByScenario(step, ScoreGETAdminTenantsBilling, scTag) + } else { + sc.AddErrorCount() + return v + } + } + + // 大会を開催、Billing確定まで進める + // _, orgAg, err := sc.GetAccountAndAgent(AccountRoleOrganizer, tenant.Name, "organizer") + // if err != nil { + // return err + // } + + // 反映まで3秒まで猶予がある + SleepWithCtx(ctx, time.Second*3) + + // 反映確認 + { + res, err := GetAdminTenantsBillingAction(ctx, billingBeforeTenantID, adminAg) + v := ValidateResponse("テナント別の請求ダッシュボード", step, res, err, WithStatusCode(200), + WithSuccessResponse(func(r ResponseAPITenantsBilling) error { + if diff := cmp.Diff(billingResultTenants, r.Data.Tenants); diff != "" { + return fmt.Errorf("AdminBillingの結果が違います (-want +got): %s", diff) + } + return nil + }), + ) + if v.IsEmpty() { + sc.AddScoreByScenario(step, ScoreGETAdminTenantsBilling, scTag) + } else { + sc.AddErrorCount() + return v + } + } + + return nil +} diff --git a/bench/scenario_peaceful_tenant.go b/bench/scenario_peaceful_tenant.go index ab171882..03907db3 100644 --- a/bench/scenario_peaceful_tenant.go +++ b/bench/scenario_peaceful_tenant.go @@ -46,9 +46,7 @@ func (sc *Scenario) PeacefulTenantScenario(ctx context.Context, step *isucandar. defer report() sc.ScenarioStart(scTag) - // TODO: 破壊的なシナリオ用IDを考える とりあえず後ろ20件 - n := ConstPeacefulTenantScenarioIDRange - index := int64((len(sc.InitialDataTenant) - n) + rand.Intn(n)) + index := int64(randomRange(ConstPeacefulTenantScenarioIDRange)) tenant := sc.InitialDataTenant[index] _, orgAg, err := sc.GetAccountAndAgent(AccountRoleOrganizer, tenant.TenantName, "organizer") @@ -78,7 +76,7 @@ func (sc *Scenario) PeacefulTenantScenario(ctx context.Context, step *isucandar. return v } } - n = rand.Intn(len(playerIDs) - 1) + n := rand.Intn(len(playerIDs) - 1) disqualifyPlayerID := playerIDs[n] checkerPlayerID := playerIDs[n+1] diff --git a/bench/scenario_billing_validate.go b/bench/scenario_tenant_billing_validate.go similarity index 89% rename from bench/scenario_billing_validate.go rename to bench/scenario_tenant_billing_validate.go index f90ff526..5e444a04 100644 --- a/bench/scenario_billing_validate.go +++ b/bench/scenario_tenant_billing_validate.go @@ -3,7 +3,6 @@ package bench import ( "context" "fmt" - "math/rand" "time" "github.com/isucon/isucandar" @@ -11,19 +10,19 @@ import ( "github.com/isucon/isucon12-qualify/data" ) -type billingValidateWorker struct { +type tenantBillingValidateWorker struct { worker *worker.Worker } -func (billingValidateWorker) String() string { - return "BillingValidateWorker" +func (tenantBillingValidateWorker) String() string { + return "TenantBillingValidateWorker" } -func (w *billingValidateWorker) Process(ctx context.Context) { w.worker.Process(ctx) } +func (w *tenantBillingValidateWorker) Process(ctx context.Context) { w.worker.Process(ctx) } -func (sc *Scenario) BillingValidateWorker(step *isucandar.BenchmarkStep, p int32) (*billingValidateWorker, error) { - scTag := ScenarioTagBillingValidate +func (sc *Scenario) TenantBillingValidateWorker(step *isucandar.BenchmarkStep, p int32) (*tenantBillingValidateWorker, error) { + scTag := ScenarioTagTenantBillingValidate w, err := worker.NewWorker(func(ctx context.Context, _ int) { - if err := sc.BillingValidate(ctx, step); err != nil { + if err := sc.TenantBillingValidate(ctx, step); err != nil { sc.ScenarioError(scTag, err) SleepWithCtx(ctx, SleepOnError) } @@ -36,16 +35,16 @@ func (sc *Scenario) BillingValidateWorker(step *isucandar.BenchmarkStep, p int32 } w.SetParallelism(p) - return &billingValidateWorker{ + return &tenantBillingValidateWorker{ worker: w, }, nil } // TODO: 1テナントで複数大会作成する -func (sc *Scenario) BillingValidate(ctx context.Context, step *isucandar.BenchmarkStep) error { +func (sc *Scenario) TenantBillingValidate(ctx context.Context, step *isucandar.BenchmarkStep) error { report := timeReporter("テナント請求検証シナリオ") defer report() - scTag := ScenarioTagBillingValidate + scTag := ScenarioTagTenantBillingValidate sc.ScenarioStart(scTag) _, adminAg, err := sc.GetAccountAndAgent(AccountRoleAdmin, "admin", "admin") @@ -75,7 +74,7 @@ func (sc *Scenario) BillingValidate(ctx context.Context, step *isucandar.Benchma } // player作成 - playerNum := ConstBillingValidateScenarioPlayerNum + playerNum := ConstTenantBillingValidateScenarioPlayerNum players := make(map[string]*PlayerData, playerNum) playerIDs := []string{} playerDisplayNames := make([]string, playerNum) @@ -128,8 +127,8 @@ func (sc *Scenario) BillingValidate(ctx context.Context, step *isucandar.Benchma // Billingの内訳を作成 // [0 < scored < visitor < noScored < playerNum] - scoredIndex := randomRange(0, playerNum-2) - visitorsIndex := randomRange(scoredIndex, playerNum-1) + scoredIndex := randomRange([]int{0, playerNum - 1}) + visitorsIndex := randomRange([]int{scoredIndex, playerNum - 0}) // noScoredIndex := playerNum scoredPlayers := playerIDs[:scoredIndex] // スコア登録者 @@ -253,8 +252,3 @@ func (sc *Scenario) BillingValidate(ctx context.Context, step *isucandar.Benchma return nil } - -// [start:end) -func randomRange(start, end int) int { - return start + rand.Intn(end-start) -} diff --git a/bench/scenario_util.go b/bench/scenario_util.go index 64d9d9fd..b32c6622 100644 --- a/bench/scenario_util.go +++ b/bench/scenario_util.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "math/rand" "os" "sync/atomic" "time" @@ -140,3 +141,9 @@ func SleepWithCtx(ctx context.Context, sleepTime time.Duration) { } return } + +// arg: []int{start, end} +// NOTE: start,endを範囲に含む (start <= n <= end) +func randomRange(rg []int) int { + return rg[0] + rand.Intn(rg[1]-rg[0]+1) +} diff --git a/bench/tags.go b/bench/tags.go index 93cfb309..6eb35c71 100644 --- a/bench/tags.go +++ b/bench/tags.go @@ -32,7 +32,8 @@ const ( ScenarioTagOrganizerPopularTenant ScenarioTag = "OrganizerPopularTenant" ScenarioTagOrganizerPeacefulTenant ScenarioTag = "OrganizerPeacefulTenant" ScenarioTagPlayer ScenarioTag = "Player" - ScenarioTagBillingValidate ScenarioTag = "BillingValidate" + ScenarioTagTenantBillingValidate ScenarioTag = "TenantBillingValidate" + ScenarioTagAdminBillingValidate ScenarioTag = "AdminBillingValidate" ) // ScoreTag毎の倍率 @@ -60,7 +61,8 @@ var ( ScenarioTagOrganizerPopularTenant, ScenarioTagOrganizerPeacefulTenant, ScenarioTagPlayer, - ScenarioTagBillingValidate, + ScenarioTagTenantBillingValidate, + ScenarioTagAdminBillingValidate, } ScoreTagList = []score.ScoreTag{ ScorePOSTAdminTenantsAdd, From c11767e9476f48d86492e05f551c98e91c4240c4 Mon Sep 17 00:00:00 2001 From: ToshihitoKon Date: Tue, 5 Jul 2022 21:04:35 +0900 Subject: [PATCH 09/10] =?UTF-8?q?AdminBilling=E3=81=A7=E3=82=B9=E3=82=B3?= =?UTF-8?q?=E3=82=A2=E3=82=92=E5=85=A5=E3=82=8C=E3=81=A6=E9=87=91=E9=A1=8D?= =?UTF-8?q?=E3=82=92=E7=A2=BA=E8=AA=8D=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bench/scenario.go | 6 ++-- bench/scenario_admin_billing_validate.go | 40 ++++++++++++++++++----- bench/scenario_tenant_billing_validate.go | 1 + 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/bench/scenario.go b/bench/scenario.go index a4de704a..6a95e90a 100644 --- a/bench/scenario.go +++ b/bench/scenario.go @@ -244,9 +244,9 @@ func (sc *Scenario) Load(c context.Context, step *isucandar.BenchmarkStep) error end = true case w := <-sc.WorkerCh: // workerを起動する // debug: 一つのworkerのみを立ち上げる - // if w.String() != "AdminBillingValidateWorker" { - // continue - // } + if w.String() != "AdminBillingValidateWorker" { + continue + } wg.Add(1) sc.CountWorker(w.String()) go func(w Worker) { diff --git a/bench/scenario_admin_billing_validate.go b/bench/scenario_admin_billing_validate.go index 0e0cbe47..d1267fb4 100644 --- a/bench/scenario_admin_billing_validate.go +++ b/bench/scenario_admin_billing_validate.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/google/go-cmp/cmp" "github.com/isucon/isucandar" "github.com/isucon/isucandar/worker" isuports "github.com/isucon/isucon12-qualify/webapp/go" @@ -54,7 +53,7 @@ func (sc *Scenario) AdminBillingValidate(ctx context.Context, step *isucandar.Be // 初期データからテナント選ぶ index := randomRange(ConstAdminBillingValidateScenarioIDRange) - // tenant := sc.InitialDataTenant[int64(index)] + tenant := sc.InitialDataTenant[int64(index)] // indexが含まれる区間がとれるAdminBillingのbefore var billingBeforeTenantID string @@ -86,21 +85,46 @@ func (sc *Scenario) AdminBillingValidate(ctx context.Context, step *isucandar.Be } // 大会を開催、Billing確定まで進める - // _, orgAg, err := sc.GetAccountAndAgent(AccountRoleOrganizer, tenant.Name, "organizer") - // if err != nil { - // return err - // } + _, orgAg, err := sc.GetAccountAndAgent(AccountRoleOrganizer, tenant.TenantName, "organizer") + if err != nil { + return err + } + + conf := &OrganizerJobConfig{ + orgAg: orgAg, + scTag: scTag, + tenantName: tenant.TenantName, + scoreRepeat: 1, + scoreInterval: 0, + addScoreNum: 0, + } + if err := sc.OrganizerJob(ctx, step, conf); err != nil { + return err + } // 反映まで3秒まで猶予がある SleepWithCtx(ctx, time.Second*3) // 反映確認 + + // チェック項目 + // 合計金額が増えていること + // TODO: 必要に応じて追加, ただしOrganizerJobによって増えた金額は現状取れない + sumYen := int64(0) + for _, t := range billingResultTenants { + sumYen += t.BillingYen + } + { res, err := GetAdminTenantsBillingAction(ctx, billingBeforeTenantID, adminAg) v := ValidateResponse("テナント別の請求ダッシュボード", step, res, err, WithStatusCode(200), WithSuccessResponse(func(r ResponseAPITenantsBilling) error { - if diff := cmp.Diff(billingResultTenants, r.Data.Tenants); diff != "" { - return fmt.Errorf("AdminBillingの結果が違います (-want +got): %s", diff) + resultYen := int64(0) + for _, t := range r.Data.Tenants { + resultYen += t.BillingYen + } + if resultYen <= sumYen { + return fmt.Errorf("全テナントの合計金額が正しくありません 金額は増えている必要があります (want: >%d, got:%d)", sumYen, resultYen) } return nil }), diff --git a/bench/scenario_tenant_billing_validate.go b/bench/scenario_tenant_billing_validate.go index 5e444a04..56341006 100644 --- a/bench/scenario_tenant_billing_validate.go +++ b/bench/scenario_tenant_billing_validate.go @@ -209,6 +209,7 @@ func (sc *Scenario) TenantBillingValidate(ctx context.Context, step *isucandar.B } } + // 3秒の猶予がある SleepWithCtx(ctx, time.Second*3) res, err := GetOrganizerBillingAction(ctx, orgAg) From 5f4e0d885dbcd16cf8c41ef13676c430c090d740 Mon Sep 17 00:00:00 2001 From: ToshihitoKon Date: Tue, 5 Jul 2022 21:08:43 +0900 Subject: [PATCH 10/10] =?UTF-8?q?debug=E7=94=A8=E3=81=AE=E6=9D=A1=E4=BB=B6?= =?UTF-8?q?=E5=A4=96=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bench/scenario.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bench/scenario.go b/bench/scenario.go index 6a95e90a..a4de704a 100644 --- a/bench/scenario.go +++ b/bench/scenario.go @@ -244,9 +244,9 @@ func (sc *Scenario) Load(c context.Context, step *isucandar.BenchmarkStep) error end = true case w := <-sc.WorkerCh: // workerを起動する // debug: 一つのworkerのみを立ち上げる - if w.String() != "AdminBillingValidateWorker" { - continue - } + // if w.String() != "AdminBillingValidateWorker" { + // continue + // } wg.Add(1) sc.CountWorker(w.String()) go func(w Worker) {