merge master

Signed-off-by: husharp <[email protected]>

components/playground/instance/tiproxy.go

@@ -20,7 +20,8 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/pelletier/go-toml"
+	"github.com/BurntSushi/toml"
+	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/utils"
 )
 
@@ -67,34 +68,32 @@ func (c *TiProxy) Start(ctx context.Context) error {
 	endpoints := pdEndpoints(c.pds, true)
 
 	configPath := filepath.Join(c.Dir, "config", "proxy.toml")
-	if c.ConfigPath != "" {
-		configPath = c.ConfigPath
-	}
-
-	configContent := ""
-	if b, err := os.ReadFile(configPath); err == nil {
-		configContent = string(b)
+	dir := filepath.Dir(configPath)
+	if err := utils.MkdirAll(dir, 0755); err != nil {
+		return err
 	}
 
-	config, err := toml.Load(configContent)
+	userConfig, err := unmarshalConfig(c.ConfigPath)
 	if err != nil {
 		return err
 	}
+	if userConfig == nil {
+		userConfig = make(map[string]any)
+	}
 
-	config.Set("proxy.pd-addrs", strings.Join(endpoints, ","))
-	config.Set("proxy.addr", utils.JoinHostPort(c.Host, c.Port))
-	config.Set("proxy.require-backend-tls", false)
-	config.Set("api.addr", utils.JoinHostPort(c.Host, c.StatusPort))
-
-	b, err := config.Marshal()
+	cf, err := os.Create(configPath)
 	if err != nil {
 		return err
 	}
 
-	if err := os.MkdirAll(filepath.Dir(configPath), 0755); err != nil {
-		return err
-	}
-	if err := os.WriteFile(configPath, b, 0644); err != nil {
+	enc := toml.NewEncoder(cf)
+	enc.Indent = ""
+	if err := enc.Encode(spec.MergeConfig(userConfig, map[string]any{
+		"proxy.pd-addrs":        strings.Join(endpoints, ","),
+		"proxy.addr":            utils.JoinHostPort(c.Host, c.Port),
+		"api.addr":              utils.JoinHostPort(c.Host, c.StatusPort),
+		"log.log-file.filename": c.LogFile(),
+	})); err != nil {
 		return err
 	}
 

components/playground/main.go

@@ -42,6 +42,7 @@ import (
 	logprinter "github.com/pingcap/tiup/pkg/logger/printer"
 	"github.com/pingcap/tiup/pkg/repository"
 	"github.com/pingcap/tiup/pkg/telemetry"
+	"github.com/pingcap/tiup/pkg/tui/colorstr"
 	"github.com/pingcap/tiup/pkg/utils"
 	"github.com/pingcap/tiup/pkg/version"
 	"github.com/spf13/cobra"
@@ -235,19 +236,23 @@ Examples:
 			if !semver.IsValid(options.Version) {
 				version, err := env.V1Repository().ResolveComponentVersion(spec.ComponentTiDB, options.Version)
 				if err != nil {
-					return errors.Annotate(err, fmt.Sprintf("can not expand version %s to a valid semver string", options.Version))
+					return errors.Annotate(err, fmt.Sprintf("Cannot resolve version %s to a valid semver string", options.Version))
 				}
 				// for nightly, may not use the same version for cluster
 				if options.Version == "nightly" {
 					version = "nightly"
 				}
-				fmt.Println(color.YellowString(`Using the version %s for version constraint "%s".
 
-If you'd like to use a TiDB version other than %s, cancel and retry with the following arguments:
-	Specify version manually:   tiup playground <version>
-	Specify version range:      tiup playground ^5
-	The nightly version:        tiup playground nightly
-`, version, options.Version, version))
+				if options.Version != version.String() {
+					colorstr.Fprintf(os.Stderr, `
+Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. If you'd like to use other versions:
+
+    Use exact version:      [tiup_command]tiup playground v7.1.0[reset]
+    Use version range:      [tiup_command]tiup playground ^5[reset]
+    Use nightly:            [tiup_command]tiup playground nightly[reset]
+
+`, options.Version, version.String())
+				}
 
 				options.Version = version.String()
 			}

components/playground/playground.go

@@ -889,12 +889,15 @@ func (p *Playground) waitAllTiFlashUp() {
 func (p *Playground) bindVersion(comp string, version string) (bindVersion string) {
 	switch comp {
 	case spec.ComponentTiKVCDC:
-		return p.bootOptions.TiKVCDC.Version
+		bindVersion = p.bootOptions.TiKVCDC.Version
 	case spec.ComponentTiProxy:
-		return p.bootOptions.TiProxy.Version
+		bindVersion = p.bootOptions.TiProxy.Version
 	default:
-		return version
 	}
+	if bindVersion == "" {
+		bindVersion = version
+	}
+	return
 }
 
 func (p *Playground) bootCluster(ctx context.Context, env *environment.Environment, options *BootOptions) error {

embed/templates/scripts/run_alertmanager.sh.tpl

@@ -11,7 +11,7 @@ exec > >(tee -i -a "{{.LogDir}}/alertmanager.log")
 exec 2>&1
 
 {{- if .NumaNode}}
-exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/alertmanager \
+exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/alertmanager/alertmanager \
 {{- else}}
 exec bin/alertmanager/alertmanager \
 {{- end}}

embed/templates/scripts/run_prometheus.sh.tpl

@@ -44,4 +44,9 @@ exec bin/prometheus/prometheus \
     --web.enable-admin-api \
     --log.level="info" \
     --storage.tsdb.path="{{.DataDir}}" \
+{{- if .AdditionalArgs}}
+{{- range .AdditionalArgs}}
+    {{.}} \
+{{- end}}
+{{- end}}
     --storage.tsdb.retention="{{.Retention}}"

go.mod

@@ -30,8 +30,8 @@ require (
 	github.com/juju/ansiterm v1.0.0
 	github.com/mattn/go-runewidth v0.0.14
 	github.com/minio/minio-go/v7 v7.0.52
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db
 	github.com/otiai10/copy v1.9.0
-	github.com/pelletier/go-toml v1.9.5
 	github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0
 	github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3
 	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c

go.sum

@@ -148,6 +148,8 @@ github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0
 github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU=
 github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
 github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -160,8 +162,6 @@ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/otiai10/mint v1.4.0 h1:umwcf7gbpEwf7WFzqmWwSv0CzbeMsae2u9ZvpP8j2q4=
 github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI=
-github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
-github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
 github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0=
 github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc=

pkg/cluster/api/pdapi.go

@@ -285,8 +285,8 @@ func (pc *PDClient) WaitLeader(retryOpt *utils.RetryOption) error {
 		}
 
 		// return error by default, to make the retry work
-		pc.l().Debugf("Still waitting for the PD leader to be elected")
-		return perrs.New("still waitting for the PD leader to be elected")
+		pc.l().Debugf("Still waiting for the PD leader to be elected")
+		return perrs.New("still waiting for the PD leader to be elected")
 	}, *retryOpt); err != nil {
 		return fmt.Errorf("error getting PD leader, %v", err)
 	}
@@ -444,8 +444,8 @@ func (pc *PDClient) EvictPDLeader(retryOpt *utils.RetryOption) error {
 		}
 
 		// return error by default, to make the retry work
-		pc.l().Debugf("Still waitting for the PD leader to transfer")
-		return perrs.New("still waitting for the PD leader to transfer")
+		pc.l().Debugf("Still waiting for the PD leader to transfer")
+		return perrs.New("still waiting for the PD leader to transfer")
 	}, *retryOpt); err != nil {
 		return fmt.Errorf("error evicting PD leader, %v", err)
 	}
@@ -524,7 +524,7 @@ func (pc *PDClient) EvictStoreLeader(host string, retryOpt *utils.RetryOption, c
 			return nil
 		}
 		pc.l().Infof(
-			"\t  Still waitting for %d store leaders to transfer...",
+			"\t  Still waiting for %d store leaders to transfer...",
 			leaderCount,
 		)
 
@@ -732,7 +732,7 @@ func (pc *PDClient) DelPD(name string, retryOpt *utils.RetryOption) error {
 		// check if the deleted member still present
 		for _, member := range currMembers.Members {
 			if member.Name == name {
-				return perrs.New("still waitting for the PD node to be deleted")
+				return perrs.New("still waiting for the PD node to be deleted")
 			}
 		}
 

pkg/cluster/spec/monitoring.go

@@ -60,6 +60,8 @@ type PrometheusSpec struct {
 	AdditionalScrapeConf  map[string]any         `yaml:"additional_scrape_conf,omitempty" validate:"additional_scrape_conf:ignore"`
 	ScrapeInterval        string                 `yaml:"scrape_interval,omitempty" validate:"scrape_interval:editable"`
 	ScrapeTimeout         string                 `yaml:"scrape_timeout,omitempty" validate:"scrape_timeout:editable"`
+
+	AdditionalArgs []string `yaml:"additional_args,omitempty" validate:"additional_args:ignore"`
 }
 
 // Remote prometheus remote config
@@ -220,6 +222,8 @@ func (i *MonitorInstance) InitConfig(
 		DataDir:   paths.Data[0],
 
 		NumaNode: spec.NumaNode,
+
+		AdditionalArgs: spec.AdditionalArgs,
 	}
 
 	fp := filepath.Join(paths.Cache, fmt.Sprintf("run_prometheus_%s_%d.sh", i.GetHost(), i.GetPort()))

pkg/cluster/spec/tiflash.go

@@ -537,6 +537,7 @@ func (i *TiFlashInstance) initTiFlashConfig(ctx context.Context, version string,
 		daemonConfig = `application.runAsDaemon: true`
 		markCacheSize = `mark_cache_size: 5368709120`
 	}
+
 	err = yaml.Unmarshal([]byte(fmt.Sprintf(`
 server_configs:
   tiflash:
@@ -559,11 +560,9 @@ server_configs:
     logger.errorlog: "%[2]s/tiflash_error.log"
     logger.log: "%[2]s/tiflash.log"
     logger.count: 20
-    logger.level: "debug"
     logger.size: "1000M"
     %[13]s
     raft.pd_addr: "%[9]s"
-    profiles.default.max_memory_usage: 0
     %[12]s
     %[14]s
 `,

pkg/cluster/template/scripts/monitoring.go

@@ -34,6 +34,8 @@ type PrometheusScript struct {
 	LogDir    string
 
 	NumaNode string
+
+	AdditionalArgs []string
 }
 
 // ConfigToFile write config content to specific path

pkg/exec/run.go

@@ -21,25 +21,32 @@ import (
 	"os/signal"
 	"path/filepath"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
-	"github.com/fatih/color"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tiup/pkg/environment"
 	"github.com/pingcap/tiup/pkg/localdata"
 	"github.com/pingcap/tiup/pkg/telemetry"
+	"github.com/pingcap/tiup/pkg/tui/colorstr"
 	"github.com/pingcap/tiup/pkg/utils"
 	"github.com/pingcap/tiup/pkg/version"
 	"golang.org/x/mod/semver"
 )
 
+// Skip displaying "Starting component ..." message for some commonly used components.
+var skipStartingMessages = map[string]bool{
+	"playground": true,
+	"cluster":    true,
+}
+
 // RunComponent start a component and wait it
 func RunComponent(env *environment.Environment, tag, spec, binPath string, args []string) error {
 	component, version := environment.ParseCompVersion(spec)
 
 	if version == "" {
-		cmdCheckUpdate(component, version, 2)
+		cmdCheckUpdate(component, version)
 	}
 
 	binPath, err := PrepareBinary(component, version, binPath)
@@ -74,7 +81,10 @@ func RunComponent(env *environment.Environment, tag, spec, binPath string, args
 		return err
 	}
 
-	fmt.Fprintf(os.Stderr, "Starting component `%s`: %s\n", component, strings.Join(c.Args, " "))
+	if skip, ok := skipStartingMessages[component]; !skip || !ok {
+		colorstr.Fprintf(os.Stderr, "Starting component [bold]%s[reset]: %s\n", component, strings.Join(c.Args, " "))
+	}
+
 	err = c.Start()
 	if err != nil {
 		return errors.Annotatef(err, "Failed to start component `%s`", component)
@@ -170,37 +180,58 @@ func PrepareCommand(p *PrepareCommandParams) (*exec.Cmd, error) {
 	return c, nil
 }
 
-func cmdCheckUpdate(component string, version utils.Version, timeoutSec int) {
-	fmt.Fprintf(os.Stderr, "tiup is checking updates for component %s ...", component)
-	updateC := make(chan string)
-	// timeout for check update
+func cmdCheckUpdate(component string, version utils.Version) {
+	const (
+		slowTimeout   = 1 * time.Second // Timeout to display checking message
+		cancelTimeout = 2 * time.Second // Timeout to cancel the check
+	)
+
+	// This mutex is used for protecting flag as well as stdout
+	mu := sync.Mutex{}
+	isCheckFinished := false
+
+	result := make(chan string, 1)
+
 	go func() {
-		time.Sleep(time.Duration(timeoutSec) * time.Second)
-		updateC <- color.YellowString("timeout(%ds)!", timeoutSec)
+		time.Sleep(slowTimeout)
+		mu.Lock()
+		defer mu.Unlock()
+		if !isCheckFinished {
+			colorstr.Fprintf(os.Stderr, "Checking updates for component [bold]%s[reset]... ", component)
+		}
+	}()
+
+	go func() {
+		time.Sleep(cancelTimeout)
+		result <- colorstr.Sprintf("[yellow]Timedout (after %s)", cancelTimeout)
 	}()
 
 	go func() {
-		var updateInfo string
 		latestV, _, err := environment.GlobalEnv().V1Repository().LatestStableVersion(component, false)
 		if err != nil {
+			result <- ""
 			return
 		}
 		selectVer, _ := environment.GlobalEnv().SelectInstalledVersion(component, version)
 
 		if semver.Compare(selectVer.String(), latestV.String()) < 0 {
-			updateInfo = fmt.Sprint(color.YellowString(`
-A new version of %[1]s is available:
-   The latest version:         %[2]s
-   Local installed version:    %[3]s
-   Update current component:   tiup update %[1]s
-   Update all components:      tiup update --all
+			result <- colorstr.Sprintf(`
+[yellow]A new version of [bold]%[1]s[reset][yellow] is available:[reset] [red][bold]%[2]s[reset] -> [green][bold]%[3]s[reset]
+
+    To update this component:   [tiup_command]tiup update %[1]s[reset]
+    To update all components:   [tiup_command]tiup update --all[reset]
 `,
-				component, latestV.String(), selectVer.String()))
+				component, selectVer.String(), latestV.String())
 		}
-		updateC <- updateInfo
 	}()
 
-	fmt.Fprintln(os.Stderr, <-updateC)
+	s := <-result
+	mu.Lock()
+	defer mu.Unlock()
+	isCheckFinished = true
+	if len(s) > 0 {
+		fmt.Fprintln(os.Stderr, s)
+	}
 }
 
 // PrepareBinary use given binpath or download from tiup mirror