Skip to content

Commit

Permalink
revert #3908 #3935 in release 3.2.1 (#3956)
Browse files Browse the repository at this point in the history 
* Revert "fix merge override values order for helm service (#3935) (#3936)"

This reverts commit ff888d7.

* Revert "fix helm deploy job vars when only deploy image in somecase"

This reverts commit 5587a7a.

Signed-off-by: Patrick Zhao <[email protected]>

---------

Signed-off-by: Patrick Zhao <[email protected]>
  • Loading branch information
@PetrusZ
PetrusZ authored Jan 10, 2025
1 parent 7d62c49 commit 1fb264b
Show file tree
Hide file tree
Showing 9 changed files with 116 additions and 61 deletions.
10 changes: 4 additions & 6 deletions pkg/microservice/aslan/core/common/repository/models/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,7 @@ type CreateFromChartTemplate struct {
YamlData *templatemodels.CustomYaml `bson:"yaml_data,omitempty" json:"yaml_data,omitempty"`
TemplateName string `bson:"template_name" json:"template_name"`
ServiceName string `bson:"service_name" json:"service_name"`
// custom variables in chart template
Variables []*Variable `bson:"variables" json:"variables"`
Variables []*Variable `bson:"variables" json:"variables"`
}

type CreateFromChartRepo struct {
Expand Down Expand Up @@ -197,10 +196,9 @@ type ServiceTmplRevision struct {
}

type HelmChart struct {
Name string `bson:"name" json:"name"`
Repo string `bson:"repo" json:"repo"`
Version string `bson:"version" json:"version"`
// full values yaml in service
Name string `bson:"name" json:"name"`
Repo string `bson:"repo" json:"repo"`
Version string `bson:"version" json:"version"`
ValuesYaml string `bson:"values_yaml" json:"values_yaml"`
}

Expand Down
7 changes: 2 additions & 5 deletions pkg/microservice/aslan/core/common/repository/models/template/product.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,10 +121,6 @@ func (grc *GitRepoConfig) GetNamespace() string {
}

type CustomYaml struct {
// helm:
// - in service: user override's values in advance setting
// - in env: user input's override yaml
// k8s: variable yaml
YamlContent string `bson:"yaml_content" json:"yaml_content"`
RenderVariableKVs []*commontypes.RenderVariableKV `bson:"render_variable_kvs" json:"render_variable_kvs"`
Source string `bson:"source" json:"source"`
Expand All @@ -143,11 +139,12 @@ type ServiceRender struct {
ChartRepo string `bson:"chart_repo,omitempty" json:"chart_repo,omitempty"`
ChartName string `bson:"chart_name,omitempty" json:"chart_name,omitempty"`
ChartVersion string `bson:"chart_version,omitempty" json:"chart_version,omitempty"`
ValuesYaml string `bson:"values_yaml,omitempty" json:"values_yaml,omitempty"` // full helm service values yaml, only record, not actually used in calculation
ValuesYaml string `bson:"values_yaml,omitempty" json:"values_yaml,omitempty"`
OverrideValues string `bson:"override_values,omitempty" json:"override_values,omitempty"` // used for helm services, json-encoded string of kv value
// ---- for helm services end ----

// OverrideYaml will be used in both helm and k8s projects
// In k8s this is variable_yaml
OverrideYaml *CustomYaml `bson:"override_yaml,omitempty" json:"override_yaml,omitempty"`
}

Expand Down
9 changes: 3 additions & 6 deletions pkg/microservice/aslan/core/common/service/kube/apply.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -790,9 +790,6 @@ func PrepareHelmServiceData(applyParam *ResourceApplyParam) (*commonmodels.Produ
OverrideYaml: &template.CustomYaml{},
}
productService.Render = targetChart
} else {
productService.GetServiceRender().ValuesYaml = svcTemplate.HelmChart.ValuesYaml
productService.GetServiceRender().ChartVersion = svcTemplate.HelmChart.Version
}

if applyParam.UpdateServiceRevision && productService.Revision != svcTemplate.Revision {
Expand All @@ -808,18 +805,18 @@ func PrepareHelmServiceData(applyParam *ResourceApplyParam) (*commonmodels.Produ
}
}

imageValuesMaps := make([]map[string]interface{}, 0)
replaceValuesMaps := make([]map[string]interface{}, 0)
for _, targetContainer := range productService.Containers {
// prepare image replace info
replaceValuesMap, err := commonutil.AssignImageData(targetContainer.Image, commonutil.GetValidMatchData(targetContainer.ImagePath))
if err != nil {
return nil, nil, fmt.Errorf("failed to pase image uri %s/%s, err %s", productInfo.ProductName, applyParam.ServiceName, err.Error())
}
imageValuesMaps = append(imageValuesMaps, replaceValuesMap)
replaceValuesMaps = append(replaceValuesMaps, replaceValuesMap)
}

// replace image into service's values.yaml
replacedValuesYaml, err := commonutil.ReplaceImage(svcTemplate.HelmChart.ValuesYaml, imageValuesMaps...)
replacedValuesYaml, err := commonutil.ReplaceImage(svcTemplate.HelmChart.ValuesYaml, replaceValuesMaps...)
if err != nil {
return nil, nil, fmt.Errorf("failed to replace image uri %s/%s, err %s", productInfo.ProductName, applyParam.ServiceName, err.Error())

Expand Down
14 changes: 5 additions & 9 deletions pkg/microservice/aslan/core/common/service/kube/helm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,11 +144,7 @@ func InstallOrUpgradeHelmChartWithValues(param *ReleaseInstallParam, isRetry boo
}

// GeneMergedValues generate values.yaml used to install or upgrade helm chart, like param in after option -f
// productSvc: contains current images info
// svcRender: contains env values info, including service's values and env's override values
// defaultValues: global values yaml
// images: images to be replaced
// fullValues: If fullValues is set to true, full values yaml content will be returned, this case is used to preview values when running workflows
// If fullValues is set to true, full values yaml content will be returned, this case is used to preview values when running workflows
func GeneMergedValues(productSvc *commonmodels.ProductService, svcRender *templatemodels.ServiceRender, defaultValues string, images []string, fullValues bool) (string, error) {
serviceName := productSvc.ServiceName
var targetContainers []*commonmodels.Container
Expand All @@ -168,18 +164,18 @@ func GeneMergedValues(productSvc *commonmodels.ProductService, svcRender *templa

targetChart := svcRender

imageValuesMaps := make([]map[string]interface{}, 0)
replaceValuesMaps := make([]map[string]interface{}, 0)
for _, targetContainer := range targetContainers {
// prepare image replace info
replaceValuesMap, err := commonutil.AssignImageData(targetContainer.Image, commonutil.GetValidMatchData(targetContainer.ImagePath))
if err != nil {
return "", fmt.Errorf("failed to pase image uri %s/%s, err %s", productSvc.ProductName, serviceName, err.Error())
}
imageValuesMaps = append(imageValuesMaps, replaceValuesMap)
replaceValuesMaps = append(replaceValuesMaps, replaceValuesMap)
}

imageKVS := make([]*helmtool.KV, 0)
for _, imageSecs := range imageValuesMaps {
for _, imageSecs := range replaceValuesMaps {
for key, value := range imageSecs {
imageKVS = append(imageKVS, &helmtool.KV{
Key: key,
Expand All @@ -189,7 +185,7 @@ func GeneMergedValues(productSvc *commonmodels.ProductService, svcRender *templa
}

// replace image into service's values.yaml
replacedValuesYaml, err := commonutil.ReplaceImage(targetChart.ValuesYaml, imageValuesMaps...)
replacedValuesYaml, err := commonutil.ReplaceImage(targetChart.ValuesYaml, replaceValuesMaps...)
if err != nil {
return "", fmt.Errorf("failed to replace image uri %s/%s, err %s", productSvc.ProductName, serviceName, err.Error())

Expand Down
17 changes: 13 additions & 4 deletions pkg/microservice/aslan/core/environment/service/environment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1498,17 +1498,17 @@ func GetAffectedServices(productName, envName string, arg *K8sRendersetArg, log
func GeneEstimatedValues(productName, envName, serviceOrReleaseName, scene, format string, arg *EstimateValuesArg, isHelmChartDeploy bool, log *zap.SugaredLogger) (interface{}, error) {
var (
productSvc *commonmodels.ProductService
tmplSvc *commonmodels.Service
latestSvc *commonmodels.Service
productInfo *commonmodels.Product
err error
)

switch scene {
case usageScenarioCreateEnv:
productInfo = &commonmodels.Product{}
productSvc, tmplSvc, err = prepareEstimateDataForEnvCreation(productName, serviceOrReleaseName, arg.Production, isHelmChartDeploy, log)
productSvc, latestSvc, err = prepareEstimateDataForEnvCreation(productName, serviceOrReleaseName, arg.Production, isHelmChartDeploy, log)
default:
productSvc, tmplSvc, productInfo, err = prepareEstimateDataForEnvUpdate(productName, envName, serviceOrReleaseName, scene, arg.Production, isHelmChartDeploy, log)
productSvc, latestSvc, productInfo, err = prepareEstimateDataForEnvUpdate(productName, envName, serviceOrReleaseName, scene, arg.Production, isHelmChartDeploy, log)
}

if err != nil {
Expand All @@ -1526,6 +1526,15 @@ func GeneEstimatedValues(productName, envName, serviceOrReleaseName, scene, form

images := make([]string, 0)

curUsedSvc, err := repository.QueryTemplateService(&commonrepo.ServiceFindOption{
ServiceName: productSvc.ServiceName,
Revision: productSvc.Revision,
ProductName: productSvc.ProductName,
}, arg.Production)
if err != nil {
curUsedSvc = nil
}

mergedValues := ""
if isHelmChartDeploy {
chartRepo, err := commonrepo.NewHelmRepoColl().Find(&commonrepo.HelmRepoFindOption{RepoName: arg.ChartRepo})
Expand All @@ -1548,7 +1557,7 @@ func GeneEstimatedValues(productName, envName, serviceOrReleaseName, scene, form
return nil, e.ErrUpdateRenderSet.AddDesc(fmt.Sprintf("failed to merge override values, err %s", err))
}
} else {
containers := kube.CalculateContainer(productSvc, tmplSvc, tmplSvc.Containers, productInfo)
containers := kube.CalculateContainer(productSvc, curUsedSvc, latestSvc.Containers, productInfo)
for _, container := range containers {
images = append(images, container.Image)
}
Expand Down
101 changes: 80 additions & 21 deletions pkg/microservice/aslan/core/environment/service/version.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,6 @@ func RollbackEnvServiceVersion(ctx *internalhandler.Context, projectName, envNam
}
} else if envSvcVersion.Service.Type == setting.HelmDeployType || envSvcVersion.Service.Type == setting.HelmChartDeployType {
var svcTmpl *commonmodels.Service
imageKVS := make([]*helmtool.KV, 0)
if envSvcVersion.Service.Type == setting.HelmDeployType {
svcTmpl, err = mongodb.NewServiceColl().Find(&mongodb.ServiceFindOption{
ProductName: envSvcVersion.ProductName,
Expand All @@ -315,28 +314,88 @@ func RollbackEnvServiceVersion(ctx *internalhandler.Context, projectName, envNam
if err != nil {
return e.ErrRollbackEnvServiceVersion.AddErr(fmt.Errorf("failed to find service temlate %s/%s/%d, error: %v", envSvcVersion.EnvName, envSvcVersion.Service.ServiceName, envSvcVersion.Service.Revision, err))
}

imageValuesMaps := make([]map[string]interface{}, 0)
for _, targetContainer := range envSvcVersion.Service.Containers {
// prepare image replace info
replaceValuesMap, err := commonutil.AssignImageData(targetContainer.Image, commonutil.GetValidMatchData(targetContainer.ImagePath))
if err != nil {
return fmt.Errorf("failed to pase image uri %s/%s, err %s", envSvcVersion.ProductName, serviceName, err.Error())
}
imageValuesMaps = append(imageValuesMaps, replaceValuesMap)
}

for _, imageSecs := range imageValuesMaps {
for key, value := range imageSecs {
imageKVS = append(imageKVS, &helmtool.KV{
Key: key,
Value: value,
})
}
}
}

mergedValues, err := helmtool.MergeOverrideValues("", envSvcVersion.DefaultValues, envSvcVersion.Service.GetServiceRender().GetOverrideYaml(), envSvcVersion.Service.GetServiceRender().OverrideValues, imageKVS)
// if env.DefaultValues != "" {
// mergedValues, err := helmtool.MergeOverrideValues("", envSvcVersion.DefaultValues, envSvcVersion.Service.GetServiceRender().GetOverrideYaml(), envSvcVersion.Service.GetServiceRender().OverrideValues, nil)
// if err != nil {
// return e.ErrRollbackEnvServiceVersion.AddErr(fmt.Errorf("failed to merge service %s's override yaml %s and values %s, err: %s", envSvcVersion.Service.ServiceName, envSvcVersion.Service.GetServiceRender().GetOverrideYaml(), envSvcVersion.Service.GetServiceRender().OverrideValues, err))
// }

// mergedValuesFlatMap, err := converter.YamlToFlatMap([]byte(mergedValues))
// if err != nil {
// return e.ErrRollbackEnvServiceVersion.AddErr(fmt.Errorf("failed to convert mergedSvcValues to flatMap, err: %s", err))
// }
// defaultValuesFlatMap, err := converter.YamlToFlatMap([]byte(env.DefaultValues))
// if err != nil {
// return e.ErrRollbackEnvServiceVersion.AddErr(fmt.Errorf("failed to convert defaultValues to flatMap, err: %s", err))
// }

// // in current env's defaultValues, but not in service's mergedValues, add it to needToAddValues
// needToAddValuesFlatMap := make(map[string]interface{})
// for defaultKey, defaultValue := range defaultValuesFlatMap {
// if _, ok := mergedValuesFlatMap[defaultKey]; !ok {
// needToAddValuesFlatMap[defaultKey] = defaultValue
// }
// }

// if envSvcVersion.Service.Type == setting.HelmDeployType {
// svcTmplValuesFlatMap, err := converter.YamlToFlatMap([]byte(svcTmpl.HelmChart.ValuesYaml))
// if err != nil {
// return e.ErrRollbackEnvServiceVersion.AddErr(fmt.Errorf("failed to convert template service %s's values yaml to flatMap, err: %v", svcTmpl.ServiceName, err))
// }
// for key, value := range needToAddValuesFlatMap {
// if v, ok := svcTmplValuesFlatMap[key]; !ok {
// // in needToAddValues, but not in service's chart values, add it to mergedValues and set value from needToAddValues
// mergedValuesFlatMap[key] = value
// } else {
// // in needToAddValues, and in service's chart values, add it to mergedValues and set value from template service values
// mergedValuesFlatMap[key] = v
// }
// }
// } else if envSvcVersion.Service.Type == setting.HelmChartDeployType {
// chartRepoName := envSvcVersion.Service.GetServiceRender().ChartRepo
// chartName := envSvcVersion.Service.GetServiceRender().ChartName
// chartVersion := envSvcVersion.Service.GetServiceRender().ChartVersion
// chartRepo, err := commonrepo.NewHelmRepoColl().Find(&commonrepo.HelmRepoFindOption{RepoName: chartRepoName})
// if err != nil {
// return fmt.Errorf("failed to query chart-repo info, productName: %s, repoName: %s", env.ProductName, chartRepoName)
// }

// hClient, err := helmclient.NewClient()
// if err != nil {
// return err
// }

// valuesYaml, err := hClient.GetChartValues(commonutil.GeneHelmRepo(chartRepo), env.ProductName, serviceName, chartRepoName, chartName, chartVersion)
// if err != nil {
// return fmt.Errorf("failed to get chart values, chartRepo: %s, chartName: %s, chartVersion: %s, err %s", chartRepoName, chartName, chartVersion, err)
// }
// valuesYamlFlatMap, err := converter.YamlToFlatMap([]byte(valuesYaml))
// if err != nil {
// return e.ErrRollbackEnvServiceVersion.AddErr(fmt.Errorf("failed to convert mergedSvcValues to flatMap, err: %s", err))
// }

// for key, value := range needToAddValuesFlatMap {
// if v, ok := valuesYamlFlatMap[key]; !ok {
// // in needToAddValues, but not in service's chart values, add it to mergedValues and set value from needToAddValues
// mergedValuesFlatMap[key] = value
// } else {
// // in needToAddValues, and in service's chart values, add it to mergedValues and set value from chart values
// mergedValuesFlatMap[key] = v
// }
// }
// }

// mergedValuesByte, err := yaml.Marshal(mergedValuesFlatMap)
// if err != nil {
// return e.ErrRollbackEnvServiceVersion.AddErr(fmt.Errorf("failed to mashal mergedValuesFlatMap, err: %s", err))
// }

// envSvcVersion.Service.GetServiceRender().OverrideYaml.YamlContent = string(mergedValuesByte)
// envSvcVersion.Service.GetServiceRender().OverrideValues = ""
// } else {
mergedValues, err := helmtool.MergeOverrideValues("", envSvcVersion.DefaultValues, envSvcVersion.Service.GetServiceRender().GetOverrideYaml(), envSvcVersion.Service.GetServiceRender().OverrideValues, nil)
if err != nil {
return e.ErrRollbackEnvServiceVersion.AddErr(fmt.Errorf("failed to merge service %s's override yaml %s and values %s, err: %s", envSvcVersion.Service.ServiceName, envSvcVersion.Service.GetServiceRender().GetOverrideYaml(), envSvcVersion.Service.GetServiceRender().OverrideValues, err))
}
Expand Down
7 changes: 7 additions & 0 deletions pkg/microservice/aslan/core/service/service/helm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -836,6 +836,7 @@ func CreateOrUpdateHelmServiceFromRepo(projectName string, args *HelmServiceCrea
filePaths = createFromRepo.Paths
base = path.Join(config.S3StoragePath(), createFromRepo.Repo)

helmRenderCharts := make([]*templatemodels.ServiceRender, 0, len(filePaths))
var wg wait.Group
var mux sync.RWMutex
var serviceList []*commonmodels.Service
Expand Down Expand Up @@ -962,6 +963,12 @@ func CreateOrUpdateHelmServiceFromRepo(projectName string, args *HelmServiceCrea
return
}
serviceList = append(serviceList, svc)

helmRenderCharts = append(helmRenderCharts, &templatemodels.ServiceRender{
ServiceName: serviceName,
ChartVersion: svc.HelmChart.Version,
ValuesYaml: svc.HelmChart.ValuesYaml,
})
})
}

Expand Down
9 changes: 1 addition & 8 deletions pkg/microservice/aslan/core/workflow/service/workflow/workflow_v4.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2335,24 +2335,17 @@ func CompareHelmServiceYamlInEnv(serviceName, variableYaml, envName, projectName

if isHelmChartDeploy {
currentYaml := ""
latestYaml := ""
chartInfo := prod.GetChartDeployRenderMap()[serviceName]
if chartInfo != nil {
currentYaml, err = helmtool.MergeOverrideValues("", "", chartInfo.GetOverrideYaml(), chartInfo.OverrideValues, nil)
if err != nil {
return nil, fmt.Errorf("failed to merge override values, err: %s", err)
}

latestYaml, err = helmtool.MergeOverrideValues("", "", variableYaml, chartInfo.OverrideValues, nil)
if err != nil {
return nil, fmt.Errorf("failed to merge override values, err: %s", err)
}
}
currentYaml = strings.TrimSuffix(currentYaml, "\n")
latestYaml = strings.TrimSuffix(latestYaml, "\n")
return &GetHelmValuesDifferenceResp{
Current: currentYaml,
Latest: latestYaml,
Latest: variableYaml,
}, nil
}

Expand Down
3 changes: 1 addition & 2 deletions pkg/tool/helmclient/helmclient.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,6 @@ type KV struct {
Value interface{} `json:"value"`
}

// @note when should set valuesYaml? for return all values of the chart?
// MergeOverrideValues merge override yaml and override kvs
// defaultValues overrideYaml used for -f option
// overrideValues used for --set option
Expand All @@ -200,7 +199,7 @@ func MergeOverrideValues(valuesYaml, defaultValues, overrideYaml, overrideValues
}
}

valuesMap, err := yamlutil.MergeAndUnmarshal([][]byte{[]byte(valuesYaml), []byte(defaultValues), []byte(overrideYaml), imageRelatedValues})
valuesMap, err := yamlutil.MergeAndUnmarshal([][]byte{imageRelatedValues, []byte(valuesYaml), []byte(defaultValues), []byte(overrideYaml)})
if err != nil {
return "", err
}
Expand Down

0 comments on commit 1fb264b

Please sign in to comment.