Update changelog

Rename some functions
Check if workflow has any steps before storing it

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## UNRELEASED
 
+### ENHANCEMENTS
+
+* Refactor deployments package to be able to use different storage backends - part Two: Consul as default Deployments store implementation ([GH-530](https://github.com/ystia/yorc/issues/530))
+
 ### BUG FIXES
 
 * Yorc bootstrap on 4.0.0-M7 doesn't work unless an alternative download URL is provided for Yorc ([GH-561](https://github.com/ystia/yorc/issues/561))

deployments/artifacts.go

@@ -71,7 +71,7 @@ func GetFileArtifactsForType(ctx context.Context, deploymentID, typeName string)
 // It will first fetch artifacts from it node type and its parents and fetch artifacts for the node template itself.
 // This way artifacts from a parent type may be overridden by child types and artifacts from node type may be overridden by the node template
 func GetFileArtifactsForNode(ctx context.Context, deploymentID, nodeName string) (map[string]string, error) {
-	node, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+	node, err := getNodeTemplate(ctx, deploymentID, nodeName)
 	if err != nil {
 		return nil, err
 	}

deployments/attribute_notifications.go

@@ -293,7 +293,7 @@ func addAttributeNotifications(ctx context.Context, deploymentID, nodeName, inst
 	}
 
 	// First look at node type as instance values can't contain functions
-	node, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+	node, err := getNodeTemplate(ctx, deploymentID, nodeName)
 	if err != nil {
 		return err
 	}

deployments/capabilities.go

@@ -132,7 +132,7 @@ func getCapabilityPropertyDefinition(ctx context.Context, deploymentID, capabili
 // It returns true if a value is found false otherwise as first return parameter.
 // If the property is not found in the node then the type hierarchy is explored to find a default value.
 func GetCapabilityPropertyValue(ctx context.Context, deploymentID, nodeName, capabilityName, propertyName string, nestedKeys ...string) (*TOSCAValue, error) {
-	node, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+	node, err := getNodeTemplate(ctx, deploymentID, nodeName)
 	if err != nil {
 		return nil, err
 	}
@@ -289,7 +289,7 @@ func GetInstanceCapabilityAttributeValue(ctx context.Context, deploymentID, node
 	}
 
 	// Then look at global node level
-	node, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+	node, err := getNodeTemplate(ctx, deploymentID, nodeName)
 	if err != nil {
 		return nil, err
 	}

deployments/datatypes.go

@@ -121,7 +121,7 @@ func GetTopologyOutputType(ctx context.Context, deploymentID, outputName string)
 }
 
 func getTopologyInputOrOutputType(ctx context.Context, deploymentID, parameterName, parameterType string) (string, error) {
-	exist, paramDef, err := getParameterDefinitionStruct(ctx, deploymentID, parameterName, parameterType)
+	exist, paramDef, err := getParameterDefinition(ctx, deploymentID, parameterName, parameterType)
 	if err != nil || !exist {
 		return "", err
 	}

deployments/definition_store.go

@@ -255,7 +255,7 @@ func checkForInstancesCreation(ctx context.Context, consulStore consulutil.Consu
 
 func fixGetOperationOutput(ctx context.Context, deploymentID, nodeName string) error {
 	// Check operation outputs on node template
-	node, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+	node, err := getNodeTemplate(ctx, deploymentID, nodeName)
 	if err != nil {
 		return err
 	}
@@ -434,7 +434,7 @@ func storeOperationOutput(ctx context.Context, deploymentID, nodeName, typeName,
 		return nil
 	}
 	if isNodeImplOpe {
-		nodeTemplate, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+		nodeTemplate, err := getNodeTemplate(ctx, deploymentID, nodeName)
 		if err != nil {
 			return err
 		}
@@ -528,7 +528,7 @@ func fixAlienBlockStorages(ctx context.Context, deploymentID, nodeName string) e
 			}
 
 			// Update the compute node with new requirement
-			node, err := getNodeTemplateStruct(ctx, deploymentID, computeNodeName)
+			node, err := getNodeTemplate(ctx, deploymentID, computeNodeName)
 			if err != nil {
 				return err
 			}

deployments/definition_store_test.go

@@ -1075,7 +1075,7 @@ func testCheckCycleInNestedWorkflows(t *testing.T) {
 func testImportTopologyTemplateNodeMetadata(t *testing.T, deploymentID string) {
 	t.Parallel()
 
-	node, err := getNodeTemplateStruct(context.Background(), deploymentID, "TestCompute")
+	node, err := getNodeTemplate(context.Background(), deploymentID, "TestCompute")
 	require.NoError(t, err, "Error getting node template")
 	require.NotNil(t, node, "node TestCompute should not be nil")
 	require.NotNil(t, node.Metadata, "node TestCompute should not be nil")

deployments/inputs.go

@@ -24,7 +24,7 @@ import (
 // GetInputValue first checks if a non-empty field value exists for this input, if it doesn't then it checks for a non-empty field default.
 // If none of them exists then it returns an empty string.
 func GetInputValue(ctx context.Context, deploymentID, inputName string, nestedKeys ...string) (string, error) {
-	exist, paramDef, err := getParameterDefinitionStruct(ctx, deploymentID, inputName, "inputs")
+	exist, paramDef, err := getParameterDefinition(ctx, deploymentID, inputName, "inputs")
 	if err != nil || !exist {
 		return "", err
 	}

deployments/internal/store_workflows.go

@@ -52,6 +52,10 @@ func transformedWorkflow(workflow *tosca.Workflow) *tosca.Workflow {
 func storeWorkflows(ctx context.Context, topology tosca.Topology, deploymentID string) error {
 	kv := make([]*types.KeyValue, 0)
 	for wfName, workflow := range topology.TopologyTemplate.Workflows {
+		// no need to store empty workflow
+		if workflow.Steps == nil {
+			continue
+		}
 		wfPrefix := path.Join(consulutil.DeploymentKVPrefix, deploymentID, "workflows", wfName)
 		kv = append(kv, &types.KeyValue{
 			Key:   wfPrefix,

deployments/nodes.go

@@ -49,7 +49,7 @@ func IsNodeNotFoundError(err error) bool {
 	return ok
 }
 
-func getNodeTemplateStruct(ctx context.Context, deploymentID, nodeName string) (*tosca.NodeTemplate, error) {
+func getNodeTemplate(ctx context.Context, deploymentID, nodeName string) (*tosca.NodeTemplate, error) {
 	node := new(tosca.NodeTemplate)
 	nodePath := path.Join(consulutil.DeploymentKVPrefix, deploymentID, "topology", "nodes", nodeName)
 	exist, err := storage.GetStore(types.StoreTypeDeployment).Get(nodePath, node)
@@ -199,7 +199,7 @@ func GetHostedOnNode(ctx context.Context, deploymentID, nodeName string) (string
 }
 
 func getHostedOnNodeAndInstance(ctx context.Context, deploymentID, nodeName, instanceName string) (string, string, error) {
-	node, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+	node, err := getNodeTemplate(ctx, deploymentID, nodeName)
 	if err != nil {
 		return "", "", err
 	}
@@ -277,7 +277,7 @@ func GetNodesHostedOn(ctx context.Context, deploymentID, hostNode string) ([]str
 }
 
 func getNodeAttributeValue(ctx context.Context, deploymentID, nodeName, instanceName, attributeName, attrDataType string, nestedKeys ...string) (*TOSCAValue, error) {
-	node, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+	node, err := getNodeTemplate(ctx, deploymentID, nodeName)
 	if err != nil {
 		return nil, err
 	}
@@ -333,7 +333,7 @@ func getNodeAttributeValue(ctx context.Context, deploymentID, nodeName, instance
 // If the property is not found in the node then the type hierarchy is explored to find a default value.
 // If the property is still not found then it will explore the HostedOn hierarchy
 func GetNodePropertyValue(ctx context.Context, deploymentID, nodeName, propertyName string, nestedKeys ...string) (*TOSCAValue, error) {
-	node, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+	node, err := getNodeTemplate(ctx, deploymentID, nodeName)
 	if err != nil {
 		return nil, err
 	}
@@ -566,7 +566,7 @@ func GetNodes(ctx context.Context, deploymentID string) ([]string, error) {
 // GetNodeType returns the type of a given node identified by its name
 func GetNodeType(ctx context.Context, deploymentID, nodeName string) (string, error) {
 	var nodeType string
-	node, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+	node, err := getNodeTemplate(ctx, deploymentID, nodeName)
 	if err != nil {
 		return "", errors.Wrapf(err, "Can't get type for node %q", nodeName)
 	}
@@ -599,7 +599,7 @@ func GetNodeAttributesNames(ctx context.Context, deploymentID, nodeName string)
 	attributesSet := make(map[string]struct{})
 
 	// Look at node type
-	node, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+	node, err := getNodeTemplate(ctx, deploymentID, nodeName)
 	if err != nil {
 		return nil, err
 	}
@@ -838,7 +838,7 @@ func DoesNodeExist(ctx context.Context, deploymentID, nodeName string) (bool, er
 
 // GetNodeMetadata retrieves the related node metadata key if exists
 func GetNodeMetadata(ctx context.Context, deploymentID, nodeName, key string) (bool, string, error) {
-	node, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+	node, err := getNodeTemplate(ctx, deploymentID, nodeName)
 	if err != nil {
 		return false, "", err
 	}

deployments/operations.go

@@ -150,7 +150,7 @@ func getOperationAndInterfaceDefinitions(ctx context.Context, deploymentID, node
 
 	var interfaceDef *tosca.InterfaceDefinition
 	if nodeTemplate != "" {
-		node, err := getNodeTemplateStruct(ctx, deploymentID, nodeTemplate)
+		node, err := getNodeTemplate(ctx, deploymentID, nodeTemplate)
 		if err != nil {
 			return nil, nil, err
 		}

deployments/policies.go

@@ -25,7 +25,7 @@ import (
 	"path"
 )
 
-func getPolicyTypeStruct(ctx context.Context, deploymentID, policyTypeName string) (*tosca.PolicyType, error) {
+func getPolicyType(ctx context.Context, deploymentID, policyTypeName string) (*tosca.PolicyType, error) {
 	policyTyp := new(tosca.PolicyType)
 	err := getExpectedTypeFromName(ctx, deploymentID, policyTypeName, policyTyp)
 	if err != nil {
@@ -34,7 +34,7 @@ func getPolicyTypeStruct(ctx context.Context, deploymentID, policyTypeName strin
 	return policyTyp, nil
 }
 
-func getPolicyStruct(ctx context.Context, deploymentID, policyName string) (*tosca.Policy, error) {
+func getPolicy(ctx context.Context, deploymentID, policyName string) (*tosca.Policy, error) {
 	key := path.Join(consulutil.DeploymentKVPrefix, deploymentID, "topology", "policies", policyName)
 	policy := new(tosca.Policy)
 	exist, err := storage.GetStore(types.StoreTypeDeployment).Get(key, policy)
@@ -58,7 +58,7 @@ func GetPoliciesForType(ctx context.Context, deploymentID, policyTypeName string
 	policies := make([]string, 0)
 	for _, key := range keys {
 		policyName := path.Base(key)
-		policy, err := getPolicyStruct(ctx, deploymentID, policyName)
+		policy, err := getPolicy(ctx, deploymentID, policyName)
 		if err != nil {
 			return nil, err
 		}
@@ -102,7 +102,7 @@ func GetPoliciesForTypeAndNode(ctx context.Context, deploymentID, policyTypeName
 // It returns true if a value is found false otherwise as first return parameter.
 // If the property is not found in the policy then the type hierarchy is explored to find a default value.
 func GetPolicyPropertyValue(ctx context.Context, deploymentID, policyName, propertyName string, nestedKeys ...string) (*TOSCAValue, error) {
-	policy, err := getPolicyStruct(ctx, deploymentID, policyName)
+	policy, err := getPolicy(ctx, deploymentID, policyName)
 	if err != nil {
 		return nil, err
 	}
@@ -145,7 +145,7 @@ func GetPolicyPropertyValue(ctx context.Context, deploymentID, policyName, prope
 
 // GetPolicyType returns the type of the policy
 func GetPolicyType(ctx context.Context, deploymentID, policyName string) (string, error) {
-	policy, err := getPolicyStruct(ctx, deploymentID, policyName)
+	policy, err := getPolicy(ctx, deploymentID, policyName)
 	if err != nil {
 		return "", err
 	}
@@ -193,7 +193,7 @@ func IsTargetForPolicy(ctx context.Context, deploymentID, policyName, nodeName s
 // GetPolicyTargets retrieves the policy template targets
 // these targets are node names
 func GetPolicyTargets(ctx context.Context, deploymentID, policyName string) ([]string, error) {
-	policy, err := getPolicyStruct(ctx, deploymentID, policyName)
+	policy, err := getPolicy(ctx, deploymentID, policyName)
 	if err != nil {
 		return nil, err
 	}
@@ -203,7 +203,7 @@ func GetPolicyTargets(ctx context.Context, deploymentID, policyName string) ([]s
 // GetPolicyTargetsForType retrieves the policy type targets
 // this targets are node types
 func GetPolicyTargetsForType(ctx context.Context, deploymentID, policyTypeName string) ([]string, error) {
-	policyType, err := getPolicyTypeStruct(ctx, deploymentID, policyTypeName)
+	policyType, err := getPolicyType(ctx, deploymentID, policyTypeName)
 	if err != nil {
 		return nil, err
 	}

deployments/requirements.go

@@ -32,7 +32,7 @@ type Requirement struct {
 }
 
 func getRequirements(ctx context.Context, deploymentID, nodeName string) ([]tosca.RequirementAssignmentMap, error) {
-	node, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+	node, err := getNodeTemplate(ctx, deploymentID, nodeName)
 	if err != nil {
 		if IsNodeNotFoundError(err) {
 			return nil, nil

deployments/substitution_mappings.go

@@ -52,7 +52,7 @@ const (
 // that it is substitutable
 func isSubstitutableNode(ctx context.Context, deploymentID, nodeName string) (bool, error) {
 
-	node, err := getNodeTemplateStruct(ctx, deploymentID, nodeName)
+	node, err := getNodeTemplate(ctx, deploymentID, nodeName)
 	if err != nil {
 		return false, errors.Wrapf(err, "Can't get directives for node %q", nodeName)
 	}

deployments/topo_output.go

@@ -26,7 +26,7 @@ import (
 	"github.com/ystia/yorc/v4/helper/consulutil"
 )
 
-func getParameterDefinitionStruct(ctx context.Context, deploymentID, parameterName, parameterType string) (bool, *tosca.ParameterDefinition, error) {
+func getParameterDefinition(ctx context.Context, deploymentID, parameterName, parameterType string) (bool, *tosca.ParameterDefinition, error) {
 	valuePath := path.Join(consulutil.DeploymentKVPrefix, deploymentID, "topology", parameterType, parameterName)
 	parameterDef := new(tosca.ParameterDefinition)
 	exist, err := storage.GetStore(types.StoreTypeDeployment).Get(valuePath, parameterDef)
@@ -43,7 +43,7 @@ func GetTopologyOutputValue(ctx context.Context, deploymentID, outputName string
 		return nil, err
 	}
 
-	exist, paramDef, err := getParameterDefinitionStruct(ctx, deploymentID, outputName, "outputs")
+	exist, paramDef, err := getParameterDefinition(ctx, deploymentID, outputName, "outputs")
 	if err != nil || !exist {
 		return nil, err
 	}

server/upgradeschema/upgrade_120.go

@@ -63,7 +63,6 @@ func upgradeDeploymentsRefactoring(cfg config.Configuration) error {
 			path.Join(topologyPath, "policies"),
 			path.Join(topologyPath, "repositories"),
 			path.Join(topologyPath, "substitution_mappings"),
-			path.Join(topologyPath, "repositories"),
 			path.Join(topologyPath, "metadata"),
 			path.Join(topologyPath, "outputs"),
 			path.Join(topologyPath, "inputs"),