Merge pull request #125 from isucon/bench/impl-23

Bench/impl 23

bench/Makefile

@@ -7,7 +7,7 @@ run: bench
 	./bench
 
 prepare: bench
-	./bench -prepare-only
+	./bench -prepare-only -strict-prepare=false
 
 5s: bench
 	./bench -duration 5s
@@ -19,7 +19,7 @@ ci-with-comment: ci
 	./pr-comment ci.log
 
 ci-loadtype-light: bench
-	./bench -target-url https://t.isucon.dev -load-type light
+	./bench -target-url https://t.isucon.dev -load-type light -strict-prepare=false
 
 .PHONY: clean
 clean:

bench/action.go

@@ -21,15 +21,6 @@ func PostInitializeAction(ctx context.Context, ag *agent.Agent) (*http.Response,
 	return ag.Do(ctx, req)
 }
 
-func GetRootAction(ctx context.Context, ag *agent.Agent) (*http.Response, error) {
-	req, err := ag.GET("/")
-	if err != nil {
-		return nil, err
-	}
-
-	return ag.Do(ctx, req)
-}
-
 func PostAdminTenantsAddAction(ctx context.Context, name, displayName string, ag *agent.Agent) (*http.Response, error) {
 	form := url.Values{}
 	form.Set("name", name)

bench/cmd/bench/main.go

@@ -21,6 +21,7 @@ const (
 	DefaultInitializeRequestTimeout = time.Second * 30
 	DefaultDuration                 = time.Minute
 	DefaultLoadType                 = bench.LoadTypeDefault
+	DefaultStrictPrepare            = true
 )
 
 func main() {
@@ -41,6 +42,7 @@ func main() {
 	flag.StringVar(&option.DataDir, "data-dir", "data", "Data directory")
 	flag.BoolVar(&option.Debug, "debug", false, "Debug mode")
 	flag.StringVar(&option.LoadType, "load-type", DefaultLoadType, fmt.Sprintf("load type [%s,%s] Default: %s", bench.LoadTypeDefault, bench.LoadTypeLight, DefaultLoadType))
+	flag.BoolVar(&option.StrictPrepare, "strict-prepare", DefaultStrictPrepare, "strict prepare mode. default: true")
 
 	// コマンドライン引数のパースを実行
 	// この時点で各フィールドに値が設定されます
@@ -103,6 +105,7 @@ func main() {
 	// スコア表示
 	scenario.PrintScenarioScoreMap()
 	scenario.PrintScenarioCount()
+	scenario.PrintWorkerCount()
 	score, addition, deduction := SumScore(result)
 	bench.ContestantLogger.Printf("SCORE: %d (+%d %d)", score, addition, -deduction)
 	br := AllTagBreakdown(result)

bench/constants.go

@@ -0,0 +1,25 @@
+package bench
+
+const (
+	ConstMaxError         = 30
+	ConstMaxCriticalError = 10
+
+	// TODO: 以下ほぼすべて要調整
+
+	// NewTenantScenario
+	ConstNewTenantScenarioPlayerWorkerNum = 10 // 作成するplayer worker数
+
+	// PopularTenantScenario
+	ConstPopularTenantScenarioScoreRepeat = 2
+	ConstPopularTenantScenarioAddScoreNum = 100 // 1度のスコア入稿で増える数
+
+	// PlayerScenario
+	ConstPlayerScenarioCompetitionLoopCount = 10 // 一周でいくつ大会を見るか
+	ConstPlayerScenarioMaxPlayerCount       = 10 // 大会1つあたり何人のプレイヤー詳細を見るか(最大値)
+
+	// PeacefulTenantScenario
+	ConstPeacefulTenantScenarioIDRange = 20 // 破壊的シナリオを許容するtenantID幅 後ろn件
+
+	// BillingValidateScenario
+	ConstBillingValidateScenarioPlayerNum = 100
+)

bench/job_organizer.go

@@ -4,17 +4,20 @@ import (
 	"context"
 	"fmt"
 	"math/rand"
+	"time"
 
 	"github.com/isucon/isucandar"
 	"github.com/isucon/isucandar/agent"
 	"github.com/isucon/isucon12-qualify/data"
 )
 
 type OrganizerJobConfig struct {
-	orgAg       *agent.Agent
-	scTag       ScenarioTag
-	tenantName  string // 対象テナント
-	scoreRepeat int
+	orgAg         *agent.Agent
+	scTag         ScenarioTag
+	tenantName    string // 対象テナント
+	scoreRepeat   int
+	scoreInterval int // スコアCSVを入稿するインターバル
+	addScoreNum   int // 一度の再投稿時に増えるスコアの数
 }
 
 // 大会を作成, スコアを増やしながら入れる, 確定する
@@ -27,11 +30,13 @@ func (sc *Scenario) OrganizerJob(ctx context.Context, step *isucandar.BenchmarkS
 
 	// player一覧を取る
 	players := make(map[string]*PlayerData)
+	playerIDs := []string{}
 	{
 		res, err := GetOrganizerPlayersListAction(ctx, conf.orgAg)
 		v := ValidateResponse("テナントのプレイヤー一覧取得", step, res, err, WithStatusCode(200),
 			WithSuccessResponse(func(r ResponseAPIPlayersList) error {
 				for _, player := range r.Data.Players {
+					playerIDs = append(playerIDs, player.ID)
 					players[player.ID] = &PlayerData{
 						ID:          player.ID,
 						DisplayName: player.DisplayName,
@@ -65,18 +70,27 @@ func (sc *Scenario) OrganizerJob(ctx context.Context, step *isucandar.BenchmarkS
 	}
 
 	// 大会結果入稿
-	// TODO: 増やし方を考える 毎度全員分スコアが増えるのはやりすぎ
+	// 全員スコアが1件ある状態がスタート
 	var score ScoreRows
+	for _, player := range players {
+		score = append(score, &ScoreRow{
+			PlayerID: player.ID,
+			Score:    rand.Intn(1000),
+		})
+	}
+
 	for count := 0; count < conf.scoreRepeat; count++ {
-		for _, player := range players {
+		for i := 0; i < conf.addScoreNum; i++ {
+			index := rand.Intn(len(playerIDs))
+			player := players[playerIDs[index]]
 			score = append(score, &ScoreRow{
 				PlayerID: player.ID,
 				Score:    rand.Intn(1000),
 			})
 		}
 		csv := score.CSV()
+		AdminLogger.Printf("[%s] [tenant:%s] CSV入稿 %d回目 (rows:%d, len:%d)", conf.scTag, conf.tenantName, count+1, len(score)-1, len(csv))
 
-		AdminLogger.Printf("[%s] [tenant:%s] CSV入稿 %d回目 len(%d)", conf.scTag, conf.tenantName, count+1, len(csv))
 		res, err := PostOrganizerCompetitionScoreAction(ctx, comp.ID, []byte(csv), conf.orgAg)
 		v := ValidateResponse("大会結果CSV入稿", step, res, err, WithStatusCode(200),
 			WithSuccessResponse(func(r ResponseAPICompetitionResult) error {
@@ -96,6 +110,8 @@ func (sc *Scenario) OrganizerJob(ctx context.Context, step *isucandar.BenchmarkS
 			sc.AddCriticalCount() // OrganizerAPI 更新系はCritical Error
 			return v
 		}
+
+		SleepWithCtx(ctx, time.Millisecond*time.Duration(conf.scoreInterval))
 	}
 
 	// 大会結果確定 x 1

bench/option.go

@@ -22,6 +22,7 @@ type Option struct {
 	DataDir                  string
 	Debug                    bool
 	LoadType                 string
+	StrictPrepare            bool
 }
 
 const (
@@ -31,12 +32,13 @@ const (
 
 func (o Option) String() string {
 	return fmt.Sprintf(
-		"TargetURL: %s, TargetAddr: %s, RequestTimeout: %s, InitializeRequestTimeout: %s, LoadType: %s",
+		"TargetURL: %s, TargetAddr: %s, RequestTimeout: %s, InitializeRequestTimeout: %s, LoadType: %s, StrictPrepare: %v",
 		o.TargetURL,
 		o.TargetAddr,
 		o.RequestTimeout.String(),
 		o.InitializeRequestTimeout.String(),
 		o.LoadType,
+		o.StrictPrepare,
 	)
 }
 

bench/run_ci.sh

@@ -2,4 +2,4 @@
 
 set -eo pipefail
 rm -f ci.log
-./bench -target-addr 127.0.0.1:443 -target-url https://t.isucon.dev | tee ci.log
+./bench -target-addr 127.0.0.1:443 -target-url https://t.isucon.dev -strict-prepare=false | tee ci.log

bench/scenario.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/isucon/isucandar"
 	"github.com/isucon/isucandar/failure"
+	"github.com/k0kubun/pp/v3"
 )
 
 var (
@@ -49,7 +50,9 @@ type Scenario struct {
 
 	ScenarioScoreMap   sync.Map // map[string]*int64
 	ScenarioCountMap   map[ScenarioTag][]int
+	WorkerCountMap     map[string]int
 	ScenarioCountMutex sync.Mutex
+	WorkerCountMutex   sync.Mutex
 
 	InitialData        InitialDataRows
 	InitialDataTenant  InitialDataTenantMap
@@ -71,6 +74,9 @@ func (sc *Scenario) Prepare(ctx context.Context, step *isucandar.BenchmarkStep)
 	sc.DisqualifiedPlayer = map[string]struct{}{}
 	sc.ScenarioCountMutex = sync.Mutex{}
 
+	sc.WorkerCountMap = make(map[string]int)
+	sc.WorkerCountMutex = sync.Mutex{}
+
 	sc.ScenarioScoreMap = sync.Map{}
 	sc.ScenarioCountMap = make(map[ScenarioTag][]int)
 	for _, key := range ScenarioTagList {
@@ -79,6 +85,10 @@ func (sc *Scenario) Prepare(ctx context.Context, step *isucandar.BenchmarkStep)
 		sc.ScenarioCountMap[key] = []int{0, 0}
 	}
 
+	sc.WorkerCh = make(chan Worker, 10)
+	sc.CriticalErrorCh = make(chan struct{}, 10)
+	sc.ErrorCh = make(chan struct{}, 10)
+
 	// GET /initialize 用ユーザーエージェントの生成
 	b, err := url.Parse(sc.Option.TargetURL)
 	if err != nil {
@@ -158,10 +168,6 @@ func (sc *Scenario) Load(c context.Context, step *isucandar.BenchmarkStep) error
 	defer ContestantLogger.Println("負荷テストを終了します")
 	wg := &sync.WaitGroup{}
 
-	sc.WorkerCh = make(chan Worker, 10)
-	sc.CriticalErrorCh = make(chan struct{}, 10)
-	sc.ErrorCh = make(chan struct{}, 10)
-
 	// 最初に起動するシナリオ
 	// AdminBillingを見続けて新規テナントを追加する
 	{
@@ -201,6 +207,7 @@ func (sc *Scenario) Load(c context.Context, step *isucandar.BenchmarkStep) error
 	}
 
 	// 破壊的な変更を許容するシナリオ
+	// TODO: 未完成
 	{
 		wkr, err := sc.PeacefulTenantScenarioWorker(step, 1)
 		if err != nil {
@@ -209,6 +216,15 @@ func (sc *Scenario) Load(c context.Context, step *isucandar.BenchmarkStep) error
 		sc.WorkerCh <- wkr
 	}
 
+	// Tenant Billingの整合性をチェックするシナリオ
+	{
+		wkr, err := sc.BillingValidateWorker(step, 1)
+		if err != nil {
+			return err
+		}
+		sc.WorkerCh <- wkr
+	}
+
 	errorCount := 0
 	criticalCount := 0
 	end := false
@@ -219,10 +235,11 @@ func (sc *Scenario) Load(c context.Context, step *isucandar.BenchmarkStep) error
 			end = true
 		case w := <-sc.WorkerCh: // workerを起動する
 			wg.Add(1)
+			sc.CountWorker(w.String())
 			go func(w Worker) {
 				defer wg.Done()
 				wkr := w
-				AdminLogger.Printf("workerを増やします [%s]", wkr)
+				defer sc.CountdownWorker(ctx, wkr.String())
 				wkr.Process(ctx)
 			}(w)
 		case <-sc.ErrorCh:
@@ -232,14 +249,14 @@ func (sc *Scenario) Load(c context.Context, step *isucandar.BenchmarkStep) error
 			criticalCount++
 		}
 
-		if 30 <= errorCount {
-			AdminLogger.Printf("エラーが30件を越えたので負荷テストを打ち切ります")
+		if ConstMaxError <= errorCount {
+			AdminLogger.Printf("エラーが%d件を越えたので負荷テストを打ち切ります", ConstMaxError)
 			cancel()
 			end = true
 		}
 
-		if 10 <= criticalCount {
-			AdminLogger.Printf("Criticalなエラーが10件を越えたので負荷テストを打ち切ります")
+		if ConstMaxCriticalError <= criticalCount {
+			AdminLogger.Printf("Criticalなエラーが%d件を越えたので負荷テストを打ち切ります", ConstMaxCriticalError)
 			cancel()
 			end = true
 		}
@@ -252,3 +269,27 @@ func (sc *Scenario) Load(c context.Context, step *isucandar.BenchmarkStep) error
 
 	return nil
 }
+
+func (sc Scenario) CountWorker(name string) {
+	sc.WorkerCountMutex.Lock()
+	defer sc.WorkerCountMutex.Unlock()
+	if _, ok := sc.WorkerCountMap[name]; !ok {
+		sc.WorkerCountMap[name] = 0
+	}
+	sc.WorkerCountMap[name]++
+	AdminLogger.Printf("workerを増やします [%s](%d)", name, sc.WorkerCountMap[name])
+}
+
+func (sc Scenario) CountdownWorker(ctx context.Context, name string) {
+	// ctxが切られたら減算しない
+	if ctx.Err() != nil {
+		return
+	}
+	sc.WorkerCountMutex.Lock()
+	defer sc.WorkerCountMutex.Unlock()
+	sc.WorkerCountMap[name]--
+}
+
+func (sc *Scenario) PrintWorkerCount() {
+	AdminLogger.Printf("WorkerCount: %s", pp.Sprint(sc.WorkerCountMap))
+}

bench/scenario_admin_billing.go

@@ -25,7 +25,7 @@ func (sc *Scenario) AdminBillingScenarioWorker(step *isucandar.BenchmarkStep, p
 	w, err := worker.NewWorker(func(ctx context.Context, _ int) {
 		if err := sc.AdminBillingScenario(ctx, step, scTag); err != nil {
 			sc.ScenarioError(scTag, err)
-			time.Sleep(SleepOnError)
+			SleepWithCtx(ctx, SleepOnError)
 		}
 	},
 		worker.WithInfinityLoop(),