improve helm variables logic

Signed-off-by: Patrick Zhao <[email protected]>

pkg/microservice/aslan/core/common/repository/models/product.go

@@ -179,6 +179,7 @@ type ProductService struct {
 	RenderedYaml   string                          `bson:"rendered_yaml,omitempty"    json:"rendered_yaml,omitempty"`
 	VariableYaml   string                          `bson:"-"                          json:"variable_yaml,omitempty"`
 	VariableKVs    []*commontypes.RenderVariableKV `bson:"-"                          json:"variable_kvs,omitempty"`
+	ValuesYaml     string                          `bson:"-"                          json:"values_yaml,omitempty"`
 	Updatable      bool                            `bson:"-"                          json:"updatable"`
 	DeployStrategy string                          `bson:"-"                          json:"deploy_strategy"`
 }

pkg/microservice/aslan/core/common/repository/models/template/product.go

@@ -129,6 +129,7 @@ type CustomYaml struct {
 	RenderVariableKVs []*commontypes.RenderVariableKV `bson:"render_variable_kvs"               json:"render_variable_kvs"`
 	Source            string                          `bson:"source"                            json:"source"`
 	AutoSync          bool                            `bson:"auto_sync"                         json:"auto_sync"`
+	AutoSyncYaml      string                          `bson:"auto_sync_yaml"                    json:"auto_sync_yaml"`
 	SourceDetail      interface{}                     `bson:"source_detail"                     json:"source_detail"`
 	SourceID          string                          `bson:"source_id"                         json:"source_id"`
 }
@@ -325,6 +326,13 @@ func (rc *ServiceRender) GetOverrideYaml() string {
 	return rc.OverrideYaml.YamlContent
 }
 
+func (rc *ServiceRender) SetOverrideYaml(Yaml string) {
+	if rc.OverrideYaml == nil {
+		rc.OverrideYaml = &CustomYaml{}
+	}
+	rc.OverrideYaml.YamlContent = Yaml
+}
+
 type KV struct {
 	Key   string      `json:"key"`
 	Value interface{} `json:"value"`

pkg/microservice/aslan/core/common/repository/models/workflow_v4.go

@@ -441,7 +441,9 @@ type DeployServiceInfo struct {
 	VariableConfigs []*DeployVariableConfig         `bson:"variable_configs"                 yaml:"variable_configs,omitempty"          json:"variable_configs,omitempty"`
 	VariableKVs     []*commontypes.RenderVariableKV `bson:"variable_kvs"                     yaml:"variable_kvs"                        json:"variable_kvs"`
 	//LatestVariableKVs []*commontypes.RenderVariableKV `bson:"latest_variable_kvs"              yaml:"latest_variable_kvs"                 json:"latest_variable_kvs"`
+	// helm 和 k8s 部署均使用该字段作为yaml格式的变量
 	VariableYaml string              `bson:"variable_yaml"                    yaml:"variable_yaml"                       json:"variable_yaml"`
+	OverrideKVs  string              `bson:"override_kvs"                     yaml:"override_kvs"` // used for helm services, json-encoded string of kv value
 	UpdateConfig bool                `bson:"update_config"                    yaml:"update_config"                       json:"update_config"`
 	Updatable    bool                `bson:"-"                        yaml:"updatable"                           json:"updatable"`
 	Deployed     bool                `bson:"-"                                yaml:"deployed"                            json:"deployed"`

pkg/microservice/aslan/core/common/service/helm/helm.go

@@ -24,19 +24,23 @@ import (
 	"time"
 
 	"go.uber.org/zap"
+	"sigs.k8s.io/yaml"
 
 	"github.com/koderover/zadig/v2/pkg/microservice/aslan/config"
 	"github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models"
 	commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models"
+	templatemodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models/template"
 	commonrepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb"
 	"github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb/template"
 	fsservice "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/fs"
 	commonutil "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/util"
 	"github.com/koderover/zadig/v2/pkg/setting"
 	"github.com/koderover/zadig/v2/pkg/tool/cache"
 	"github.com/koderover/zadig/v2/pkg/tool/crypto"
+	helmtool "github.com/koderover/zadig/v2/pkg/tool/helmclient"
 	"github.com/koderover/zadig/v2/pkg/tool/log"
 	"github.com/koderover/zadig/v2/pkg/tool/mongo"
+	"github.com/koderover/zadig/v2/pkg/util/converter"
 	"github.com/pkg/errors"
 )
 
@@ -330,3 +334,149 @@ func UpdateServicesGroupInEnv(productName, envName string, index int, group []*m
 
 	return mongo.CommitTransaction(session)
 }
+
+// GeneMergedValues generate values.yaml used to install or upgrade helm chart, like param in after option -f
+// productSvc: contains current images info
+// defaultValues: global values yaml
+func NewGeneMergedValues(productSvc *commonmodels.ProductService, defaultValues string) (string, string, error) {
+	envValuesYaml := productSvc.GetServiceRender().GetOverrideYaml()
+	overrideKVs := productSvc.GetServiceRender().OverrideValues
+
+	valuesMap := make(map[string]interface{})
+	err := yaml.Unmarshal([]byte(envValuesYaml), &valuesMap)
+	if err != nil {
+		return "", "", fmt.Errorf("Failed to unmarshall yaml, err %s", err)
+	}
+
+	flatValuesMap, err := converter.Flatten(valuesMap)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to flatten values map, err: %s", err)
+	}
+
+	// 1. calc weather to add container image into values yaml
+	found := false
+	mergedContainers := []*commonmodels.Container{}
+	for _, container := range productSvc.Containers {
+		if container.ImagePath == nil {
+			return "", "", fmt.Errorf("failed to parse image for container:%s", container.Image)
+		}
+
+		imageSearchRule := &templatemodels.ImageSearchingRule{
+			Repo:      container.ImagePath.Repo,
+			Namespace: container.ImagePath.Namespace,
+			Image:     container.ImagePath.Image,
+			Tag:       container.ImagePath.Tag,
+		}
+		pattern := imageSearchRule.GetSearchingPattern()
+		imageUrl, err := commonutil.GeneImageURI(pattern, flatValuesMap)
+		if err != nil {
+			return "", "", fmt.Errorf("failed to get image url for container:%s", container.Image)
+		}
+
+		name := commonutil.ExtractImageName(imageUrl)
+
+		if container.ImageName == name {
+			// found image in values
+			found = true
+		}
+
+		if !found {
+			// if not found corresponding service module image in values
+			// add container image into values
+			log.Debugf("not found service module %s's image in values, add it", container.ImageName)
+			mergedContainers = append(mergedContainers, container)
+		}
+	}
+
+	// 2. replace image into values yaml
+	serviceName := productSvc.ServiceName
+	imageValuesMaps := make([]map[string]interface{}, 0)
+	for _, mergedContainer := range mergedContainers {
+		log.Debugf("merged container: %+v", mergedContainer)
+		// prepare image replace info
+		replaceValuesMap, err := commonutil.AssignImageData(mergedContainer.Image, commonutil.GetValidMatchData(mergedContainer.ImagePath))
+		if err != nil {
+			return "", "", fmt.Errorf("failed to pase image uri %s/%s, err %s", productSvc.ProductName, serviceName, err.Error())
+		}
+		imageValuesMaps = append(imageValuesMaps, replaceValuesMap)
+	}
+
+	log.Debugf("imageValuesMaps: %+v", imageValuesMaps)
+	replacedEnvValuesYaml, err := commonutil.ReplaceImage(envValuesYaml, imageValuesMaps...)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to replace image uri %s/%s, err %s", productSvc.ProductName, serviceName, err.Error())
+
+	}
+	if replacedEnvValuesYaml == "" {
+		return "", "", fmt.Errorf("failed to set new image uri into service's values.yaml %s/%s", productSvc.ProductName, serviceName)
+	}
+
+	// 3. merge override values and kvs into values yaml
+	finalValuesYaml, err := helmtool.MergeOverrideValues("", defaultValues, replacedEnvValuesYaml, overrideKVs, nil)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to merge override values, err: %s", err)
+	}
+	log.Debugf("replacedEnvValuesYaml: %s", replacedEnvValuesYaml)
+	log.Debugf("finalValuesYaml: %s", finalValuesYaml)
+
+	// 4. update container image in productSvc
+	valuesMap = make(map[string]interface{})
+	err = yaml.Unmarshal([]byte(finalValuesYaml), &valuesMap)
+	if err != nil {
+		return "", "", fmt.Errorf("Failed to unmarshall yaml, err %s", err)
+	}
+
+	flatValuesMap, err = converter.Flatten(valuesMap)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to flatten values map, err: %s", err)
+	}
+	for _, container := range productSvc.Containers {
+		if container.ImagePath == nil {
+			return "", "", fmt.Errorf("failed to parse image for container:%s", container.Image)
+		}
+
+		imageSearchRule := &templatemodels.ImageSearchingRule{
+			Repo:      container.ImagePath.Repo,
+			Namespace: container.ImagePath.Namespace,
+			Image:     container.ImagePath.Image,
+			Tag:       container.ImagePath.Tag,
+		}
+		pattern := imageSearchRule.GetSearchingPattern()
+		imageUrl, err := commonutil.GeneImageURI(pattern, flatValuesMap)
+		if err != nil {
+			return "", "", fmt.Errorf("failed to get image url for container:%s", container.Image)
+		}
+		log.Debugf("final image url: %s", imageUrl)
+		container.Image = imageUrl
+	}
+
+	return finalValuesYaml, replacedEnvValuesYaml, nil
+}
+
+func GeneFullValues(serviceValuesYaml, envValuesYaml string) (string, error) {
+	finalValuesYaml, err := helmtool.MergeOverrideValues(serviceValuesYaml, "", envValuesYaml, "", nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to merge override values, err: %s", err)
+	}
+
+	return finalValuesYaml, nil
+}
+
+func convertImagePathToPatterns(container *commonmodels.Container) []map[string]string {
+	patterns := make([]map[string]string, 0)
+	if container.ImagePath != nil {
+		if container.ImagePath.Repo != "" {
+			patterns = append(patterns, map[string]string{setting.PathSearchComponentRepo: container.ImagePath.Repo})
+		}
+		if container.ImagePath.Namespace != "" {
+			patterns = append(patterns, map[string]string{setting.PathSearchComponentNamespace: container.ImagePath.Namespace})
+		}
+		if container.ImagePath.Image != "" {
+			patterns = append(patterns, map[string]string{setting.PathSearchComponentImage: container.ImagePath.Image})
+		}
+		if container.ImagePath.Tag != "" {
+			patterns = append(patterns, map[string]string{setting.PathSearchComponentTag: container.ImagePath.Tag})
+		}
+	}
+	return patterns
+}

pkg/microservice/aslan/core/common/service/kube/helm.go

@@ -143,6 +143,120 @@ func InstallOrUpgradeHelmChartWithValues(param *ReleaseInstallParam, isRetry boo
 	return err
 }
 
+// // GeneMergedValues generate values.yaml used to install or upgrade helm chart, like param in after option -f
+// // productSvc: contains current images info
+// // defaultValues: global values yaml
+// // images: images to be replaced
+// // fullValues: If fullValues is set to true, full values yaml content will be returned, this case is used to preview values when running workflows
+// func NewGeneMergedValues(templateSvc *commonmodels.Service, productSvc *commonmodels.ProductService, defaultValues string, fullValues bool) (string, string, error) {
+// 	serviceValuesYaml := templateSvc.HelmChart.ValuesYaml
+// 	defaultValues = defaultValues
+// 	envValuesYaml := productSvc.GetServiceRender().GetOverrideYaml()
+// 	overrideKVs := productSvc.GetServiceRender().OverrideValues
+// 	// images: deprecated
+// 	fullValues = fullValues
+
+// 	valuesMap := make(map[string]interface{})
+// 	err := yaml.Unmarshal([]byte(envValuesYaml), &valuesMap)
+// 	if err != nil {
+// 		err = fmt.Errorf("Failed to unmarshall yaml, err %s", err)
+// 		return "", "", err
+// 	}
+
+// 	flatValuesMap, err := converter.Flatten(valuesMap)
+// 	if err != nil {
+// 		return "", "", err
+// 	}
+
+// 	mergedContainers := []*commonmodels.Container{}
+// 	for _, container := range productSvc.Containers {
+// 		patterns := convertImagePathToPatterns(container)
+
+// 		matchedPath, err := yamlutil.SearchByPattern(flatValuesMap, patterns)
+// 		if err != nil {
+// 			return "", "", err
+// 		}
+
+// 		found := false
+// 		usedImagePath := sets.NewString()
+// 		for _, searchResult := range matchedPath {
+// 			uniquePath := commonutil.GenerateUniquePath(searchResult)
+// 			if usedImagePath.Has(uniquePath) {
+// 				continue
+// 			}
+// 			usedImagePath.Insert(uniquePath)
+// 			imageUrl, err := commonutil.GeneImageURI(searchResult, flatValuesMap)
+// 			if err != nil {
+// 				return "", "", err
+// 			}
+// 			name := commonutil.ExtractImageName(imageUrl)
+
+// 			if container.ImageName == name {
+// 				// found image in values
+// 				found = true
+// 			}
+// 		}
+
+// 		if !found {
+// 			// if not found corresponding service module image in values
+// 			// add container image into values
+// 			mergedContainers = append(mergedContainers, container)
+// 		}
+// 	}
+
+// 	serviceName := productSvc.ServiceName
+// 	imageValuesMaps := make([]map[string]interface{}, 0)
+// 	for _, mergedContainer := range mergedContainers {
+// 		// prepare image replace info
+// 		replaceValuesMap, err := commonutil.AssignImageData(mergedContainer.Image, commonutil.GetValidMatchData(mergedContainer.ImagePath))
+// 		if err != nil {
+// 			return "", "", fmt.Errorf("failed to pase image uri %s/%s, err %s", productSvc.ProductName, serviceName, err.Error())
+// 		}
+// 		imageValuesMaps = append(imageValuesMaps, replaceValuesMap)
+// 	}
+
+// 	// replace image into service's values.yaml
+// 	replacedEnvValuesYaml, err := commonutil.ReplaceImage(envValuesYaml, imageValuesMaps...)
+// 	if err != nil {
+// 		return "", "", fmt.Errorf("failed to replace image uri %s/%s, err %s", productSvc.ProductName, serviceName, err.Error())
+
+// 	}
+// 	if replacedEnvValuesYaml == "" {
+// 		return "", "", fmt.Errorf("failed to set new image uri into service's values.yaml %s/%s", productSvc.ProductName, serviceName)
+// 	}
+
+// 	baseValuesYaml := ""
+// 	if fullValues {
+// 		baseValuesYaml = serviceValuesYaml
+// 	}
+
+// 	// merge override values and kvs into service's yaml
+// 	finalValuesYaml, err := helmtool.MergeOverrideValues(baseValuesYaml, defaultValues, replacedEnvValuesYaml, overrideKVs, nil)
+// 	if err != nil {
+// 		return "", "", fmt.Errorf("failed to merge override values, err: %s", err)
+// 	}
+// 	return finalValuesYaml, replacedEnvValuesYaml, nil
+// }
+
+// func convertImagePathToPatterns(container *commonmodels.Container) []map[string]string {
+// 	patterns := make([]map[string]string, 0)
+// 	if container.ImagePath != nil {
+// 		if container.ImagePath.Repo != "" {
+// 			patterns = append(patterns, map[string]string{setting.PathSearchComponentRepo: container.ImagePath.Repo})
+// 		}
+// 		if container.ImagePath.Namespace != "" {
+// 			patterns = append(patterns, map[string]string{setting.PathSearchComponentNamespace: container.ImagePath.Namespace})
+// 		}
+// 		if container.ImagePath.Image != "" {
+// 			patterns = append(patterns, map[string]string{setting.PathSearchComponentImage: container.ImagePath.Image})
+// 		}
+// 		if container.ImagePath.Tag != "" {
+// 			patterns = append(patterns, map[string]string{setting.PathSearchComponentTag: container.ImagePath.Tag})
+// 		}
+// 	}
+// 	return patterns
+// }
+
 // GeneMergedValues generate values.yaml used to install or upgrade helm chart, like param in after option -f
 // productSvc: contains current images info
 // svcRender: contains env values info, including service's values and env's override values

pkg/microservice/aslan/core/common/service/service.go

@@ -132,6 +132,7 @@ type EnvService struct {
 	ServiceName        string                          `json:"service_name"`
 	ServiceModules     []*commonmodels.Container       `json:"service_modules"`
 	VariableYaml       string                          `json:"variable_yaml"`
+	OverrideKVs        string                          `json:"override_kvs"`
 	VariableKVs        []*commontypes.RenderVariableKV `json:"variable_kvs"`
 	LatestVariableYaml string                          `json:"latest_variable_yaml"`
 	LatestVariableKVs  []*commontypes.RenderVariableKV `json:"latest_variable_kvs"`
@@ -1187,6 +1188,7 @@ func BuildServiceInfoInEnv(productInfo *commonmodels.Product, templateSvcs []*co
 			}
 		} else if deployType == setting.HelmDeployType {
 			svc.VariableYaml = productInfo.GetSvcRender(serviceName).OverrideYaml.YamlContent
+			svc.OverrideKVs = productInfo.GetSvcRender(serviceName).OverrideValues
 		}
 
 		ret.Services = append(ret.Services, svc)

pkg/microservice/aslan/core/common/util/service.go

@@ -355,7 +355,7 @@ func parseImagesByPattern(nested map[string]interface{}, patterns []map[string]s
 	ret := make([]*commonmodels.Container, 0)
 	usedImagePath := sets.NewString()
 	for _, searchResult := range matchedPath {
-		uniquePath := generateUniquePath(searchResult)
+		uniquePath := GenerateUniquePath(searchResult)
 		if usedImagePath.Has(uniquePath) {
 			continue
 		}
@@ -388,6 +388,7 @@ func ParseImagesByRules(nested map[string]interface{}, matchRules []*templatemod
 		}
 		patterns = append(patterns, rule.GetSearchingPattern())
 	}
+	log.Debugf("nested: %v\n, patterns: %v", nested, patterns)
 	return parseImagesByPattern(nested, patterns)
 }
 
@@ -411,7 +412,7 @@ func GetPresetRules() []*templatemodels.ImageSearchingRule {
 	return ret
 }
 
-func generateUniquePath(pathData map[string]string) string {
+func GenerateUniquePath(pathData map[string]string) string {
 	keys := []string{setting.PathSearchComponentRepo, setting.PathSearchComponentNamespace, setting.PathSearchComponentImage, setting.PathSearchComponentTag}
 	values := make([]string, 0)
 	for _, key := range keys {