Merge pull request #532 from LBGarber/filter-regex-sub

Add regex and substrings to filterable data sources

linode/acceptance/test_util.go

@@ -26,6 +26,8 @@ import (
 const optInTestsEnvVar = "ACC_OPT_IN_TESTS"
 const SkipInstanceReadyPollKey = "skip_instance_ready_poll"
 
+type AttrValidateFunc func(val string) error
+
 var (
 	optInTests         map[string]struct{}
 	privateKeyMaterial string
@@ -163,13 +165,49 @@ func CheckResourceAttrContains(resName string, path, desiredValue string) resour
 		}
 
 		if !strings.Contains(value, desiredValue) {
-			return fmt.Errorf("value was not found")
+			return fmt.Errorf("value '%s' was not found", desiredValue)
+		}
+
+		return nil
+	}
+}
+
+func ValidateResourceAttr(resName, path string, comparisonFunc AttrValidateFunc) resource.TestCheckFunc {
+	return func(s *terraform.State) error {
+		rs, ok := s.RootModule().Resources[resName]
+		if !ok {
+			return fmt.Errorf("Not found: %s", resName)
+		}
+
+		value, ok := rs.Primary.Attributes[path]
+		if !ok {
+			return fmt.Errorf("attribute %s does not exist", path)
+		}
+
+		err := comparisonFunc(value)
+		if err != nil {
+			return fmt.Errorf("comparison failed: %s", err)
 		}
 
 		return nil
 	}
 }
 
+func CheckResourceAttrGreaterThan(resName, path string, target int) resource.TestCheckFunc {
+	return ValidateResourceAttr(resName, path, func(val string) error {
+		valInt, err := strconv.Atoi(val)
+		if err != nil {
+			return err
+		}
+
+		if !(valInt > target) {
+			return fmt.Errorf("%d <= %d", valInt, target)
+		}
+
+		return nil
+	})
+}
+
 func CheckResourceAttrNotEqual(resName string, path, notValue string) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		rs, ok := s.RootModule().Resources[resName]

linode/helper/filter.go

@@ -1,10 +1,15 @@
 package helper
 
 import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
-
-	"encoding/json"
+	"golang.org/x/crypto/sha3"
 )
 
 // FilterTypeFunc is a function that takes in a filter name and value,
@@ -31,6 +36,14 @@ func FilterSchema(validFilters []string) *schema.Schema {
 					Description: "The value(s) to be used in the filter.",
 					Required:    true,
 				},
+				"match_by": {
+					Type:        schema.TypeString,
+					Description: "The type of comparison to use for this filter.",
+					Optional:    true,
+					Default:     "exact",
+					ValidateFunc: validation.StringInSlice([]string{"exact", "substring", "sub", "re", "regex"},
+						false),
+				},
 			},
 		},
 	}
@@ -57,6 +70,23 @@ func OrderSchema() *schema.Schema {
 	}
 }
 
+// GetFilterID creates a unique ID specific to the current filter data source
+func GetFilterID(d *schema.ResourceData) (string, error) {
+	idMap := map[string]interface{}{
+		"filter":   d.Get("filter"),
+		"order":    d.Get("order"),
+		"order_by": d.Get("order_by"),
+	}
+
+	result, err := json.Marshal(idMap)
+	if err != nil {
+		return "", err
+	}
+
+	hash := sha3.Sum512(result)
+	return base64.StdEncoding.EncodeToString(hash[:]), nil
+}
+
 // ConstructFilterString constructs a Linode filter JSON string from each filter element in the schema
 func ConstructFilterString(d *schema.ResourceData, typeFunc FilterTypeFunc) (string, error) {
 	filters := d.Get("filter").([]interface{})
@@ -73,6 +103,12 @@ func ConstructFilterString(d *schema.ResourceData, typeFunc FilterTypeFunc) (str
 
 		name := filter["name"].(string)
 		values := filter["values"].([]interface{})
+		matchBy := filter["match_by"].(string)
+
+		// Defer this logic to the client
+		if matchBy != "exact" {
+			continue
+		}
 
 		subFilter := make([]interface{}, len(values))
 
@@ -93,6 +129,10 @@ func ConstructFilterString(d *schema.ResourceData, typeFunc FilterTypeFunc) (str
 		})
 	}
 
+	if len(rootFilter) < 1 {
+		return "{}", nil
+	}
+
 	resultMap["+and"] = rootFilter
 
 	if orderBy, ok := d.GetOk("order_by"); ok {
@@ -107,3 +147,115 @@ func ConstructFilterString(d *schema.ResourceData, typeFunc FilterTypeFunc) (str
 
 	return string(result), nil
 }
+
+// FilterResults filters the given results on the client-side filters present in the resource
+func FilterResults(d *schema.ResourceData, items []interface{}) ([]map[string]interface{}, error) {
+	result := make([]map[string]interface{}, 0)
+
+	for _, item := range items {
+		item := item.(map[string]interface{})
+
+		match, err := itemMatchesFilter(d, item)
+		if err != nil {
+			return nil, err
+		}
+
+		if !match {
+			continue
+		}
+
+		result = append(result, item)
+	}
+
+	return result, nil
+}
+
+func itemMatchesFilter(d *schema.ResourceData, item map[string]interface{}) (bool, error) {
+	filters := d.Get("filter").([]interface{})
+
+	for _, filter := range filters {
+		filter := filter.(map[string]interface{})
+
+		name := filter["name"].(string)
+		values := filter["values"].([]interface{})
+		matchBy := filter["match_by"].(string)
+
+		if matchBy == "exact" {
+			continue
+		}
+
+		itemValue, ok := item[name]
+		if !ok {
+			return false, fmt.Errorf("\"%v\" is not a valid attribute", name)
+		}
+
+		valid, err := validateFilter(matchBy, name, ExpandStringList(values), itemValue)
+		if err != nil {
+			return false, err
+		}
+
+		if !valid {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func validateFilter(matchBy, name string, values []string, itemValue interface{}) (bool, error) {
+	// Filter recursively on lists (tags, etc.)
+	if items, ok := itemValue.([]string); ok {
+		for _, item := range items {
+			valid, err := validateFilter(matchBy, name, values, item)
+			if err != nil {
+				return false, err
+			}
+
+			if valid {
+				return true, nil
+			}
+		}
+
+		return false, nil
+	}
+
+	// Only string attributes should be considered
+	itemValueStr, ok := itemValue.(string)
+	if !ok {
+		return false, fmt.Errorf("\"%s\" is not a string", name)
+	}
+
+	switch matchBy {
+	case "substring", "sub":
+		return validateFilterSubstring(values, itemValueStr)
+	case "re", "regex":
+		return validateFilterRegex(values, itemValueStr)
+	}
+
+	return true, nil
+}
+
+func validateFilterSubstring(values []string, result string) (bool, error) {
+	for _, value := range values {
+		if strings.Contains(result, value) {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func validateFilterRegex(values []string, result string) (bool, error) {
+	for _, value := range values {
+		r, err := regexp.Compile(value)
+		if err != nil {
+			return false, fmt.Errorf("failed to compile regex: %s", err)
+		}
+
+		if r.MatchString(result) {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}

linode/images/datasource.go

@@ -24,6 +24,11 @@ func readDataSource(ctx context.Context, d *schema.ResourceData, meta interface{
 
 	latestFlag := d.Get("latest").(bool)
 
+	filterID, err := helper.GetFilterID(d)
+	if err != nil {
+		return diag.Errorf("failed to generate filter id: %s", err)
+	}
+
 	filter, err := helper.ConstructFilterString(d, imageValueToFilterType)
 	if err != nil {
 		return diag.Errorf("failed to construct filter: %s", err)
@@ -32,26 +37,30 @@ func readDataSource(ctx context.Context, d *schema.ResourceData, meta interface{
 	images, err := client.ListImages(ctx, &linodego.ListOptions{
 		Filter: filter,
 	})
-
 	if err != nil {
 		return diag.Errorf("failed to list linode images: %s", err)
 	}
 
+	imagesFlattened := make([]interface{}, len(images))
+	for i, image := range images {
+		imagesFlattened[i] = flattenImage(&image)
+	}
+
+	imagesFiltered, err := helper.FilterResults(d, imagesFlattened)
+	if err != nil {
+		return diag.Errorf("failed to filter returned images: %s", err)
+	}
+
 	if latestFlag {
-		latestImage := getLatestImage(images)
+		latestImage := getLatestImage(imagesFiltered)
 
 		if latestImage != nil {
-			images = []linodego.Image{*latestImage}
+			imagesFiltered = []map[string]interface{}{latestImage}
 		}
 	}
 
-	imagesFlattened := make([]interface{}, len(images))
-	for i, image := range images {
-		imagesFlattened[i] = flattenImage(&image)
-	}
-
-	d.SetId(filter)
-	d.Set("images", imagesFlattened)
+	d.SetId(filterID)
+	d.Set("images", imagesFiltered)
 
 	return nil
 }
@@ -93,20 +102,28 @@ func imageValueToFilterType(filterName, value string) (interface{}, error) {
 	return value, nil
 }
 
-func getLatestImage(images []linodego.Image) *linodego.Image {
-	var result *linodego.Image
+func getLatestImage(images []map[string]interface{}) map[string]interface{} {
+	var latestCreated time.Time
+	var latestImage map[string]interface{}
 
 	for _, image := range images {
-		if image.Created == nil {
+		created, ok := image["created"]
+		if !ok {
 			continue
 		}
 
-		if result != nil && !image.Created.After(*result.Created) {
+		createdTime, err := time.Parse(time.RFC3339, created.(string))
+		if err != nil {
+			return nil
+		}
+
+		if latestImage != nil && !createdTime.After(latestCreated) {
 			continue
 		}
 
-		result = &image
+		latestCreated = createdTime
+		latestImage = image
 	}
 
-	return result
+	return latestImage
 }

linode/images/datasource_test.go

@@ -70,8 +70,15 @@ func TestAccDataSourceImages_basic(t *testing.T) {
 				Check: resource.ComposeTestCheckFunc(
 					// Ensure order is correctly appended to filter
 					resource.TestCheckResourceAttr(resourceName, "images.#", "2"),
-					acceptance.CheckResourceAttrContains(resourceName, "id", "\"+order_by\":\"size\""),
-					acceptance.CheckResourceAttrContains(resourceName, "id", "\"+order\":\"desc\""),
+				),
+			},
+
+			{
+				Config: tmpl.DataSubstring(t, imageName),
+				Check: resource.ComposeTestCheckFunc(
+					// Ensure order is correctly appended to filter
+					acceptance.CheckResourceAttrGreaterThan(resourceName, "images.#", 1),
+					acceptance.CheckResourceAttrContains(resourceName, "images.0.label", "Alpine"),
 				),
 			},
 		},

linode/images/tmpl/data_latest.gotf

@@ -8,6 +8,7 @@ data "linode_images" "foobar" {
     filter {
         name = "label"
         values = [linode_image.foobar.label]
+        match_by = "substring"
     }
 
     filter {

linode/images/tmpl/data_substring.gotf

@@ -0,0 +1,18 @@
+{{ define "images_data_substring" }}
+
+{{ template "images_data_base" . }}
+
+data "linode_images" "foobar" {
+    filter {
+        name = "label"
+        values = ["Alpine"]
+        match_by = "substring"
+    }
+
+    filter {
+        name = "is_public"
+        values = [true]
+    }
+}
+
+{{ end }}

linode/images/tmpl/template.go

@@ -29,3 +29,8 @@ func DataOrder(t *testing.T, image string) string {
 	return acceptance.ExecuteTemplate(t,
 		"images_data_order", TemplateData{Image: image})
 }
+
+func DataSubstring(t *testing.T, image string) string {
+	return acceptance.ExecuteTemplate(t,
+		"images_data_substring", TemplateData{Image: image})
+}