From 54dc1518d786eb7d6ca4ac9e1bfbaf4f00a0c1b8 Mon Sep 17 00:00:00 2001 From: Pierre Mavro Date: Thu, 19 Nov 2020 23:58:06 +0100 Subject: [PATCH] feat: adding Elasticache support --- charts/pleco/Chart.yaml | 4 +- charts/pleco/values.yaml | 2 +- cmd/version.go | 2 +- core/daemon.go | 10 ++- providers/aws/documentdb.go | 12 +-- providers/aws/elasticache.go | 148 +++++++++++++++++++++++++++++++++++ providers/aws/rds.go | 10 +-- 7 files changed, 171 insertions(+), 17 deletions(-) create mode 100644 providers/aws/elasticache.go diff --git a/charts/pleco/Chart.yaml b/charts/pleco/Chart.yaml index 1e40177..34c672e 100644 --- a/charts/pleco/Chart.yaml +++ b/charts/pleco/Chart.yaml @@ -3,6 +3,6 @@ name: pleco description: Automatically removes Cloud managed services and Kubernetes resources based on tags with TTL type: application home: https://github.com/Qovery/pleco -version: 0.1.8 -appVersion: 0.1.8 +version: 0.2.0 +appVersion: 0.2.0 icon: https://github.com/Qovery/pleco/raw/main/assets/pleco_logo.png diff --git a/charts/pleco/values.yaml b/charts/pleco/values.yaml index f556f9a..0deef9f 100644 --- a/charts/pleco/values.yaml +++ b/charts/pleco/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 image: repository: qoveryrd/pleco pullPolicy: IfNotPresent - plecoImageTag: "v0.1.8" + plecoImageTag: "v0.2.0" environmentVariables: CHECK_INTERVAL: "120" diff --git a/cmd/version.go b/cmd/version.go index 9467f1d..9d5c8b0 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -19,5 +19,5 @@ func init() { } func GetCurrentVersion() string { - return "0.1.8" // ci-version-check + return "0.2.0" // ci-version-check } \ No newline at end of file diff --git a/core/daemon.go b/core/daemon.go index 1f4215d..ea82edb 100644 --- a/core/daemon.go +++ b/core/daemon.go @@ -23,15 +23,21 @@ func StartDaemon(dryRun bool, interval int64) { } currentRdsSession := aws.RdsSession(*currentSession, os.Getenv("AWS_DEFAULT_REGION")) + currentElasticacheSession := aws.ElasticacheSession(*currentSession, os.Getenv("AWS_DEFAULT_REGION")) for { // check RDS - err = aws.DeleteExpiredDatabases(*currentRdsSession, "ttl", dryRun) + err = aws.DeleteExpiredRDSDatabases(*currentRdsSession, "ttl", dryRun) if err != nil { log.Error(err) } // check DocumentDB - err = aws.DeleteExpiredClusters(*currentRdsSession, "ttl", dryRun) + err = aws.DeleteExpiredDocumentDBClusters(*currentRdsSession, "ttl", dryRun) + if err != nil { + log.Error(err) + } + // check Elasticache + err = aws.DeleteExpiredElasticacheDatabases(*currentElasticacheSession, "ttl", dryRun) if err != nil { log.Error(err) } diff --git a/providers/aws/documentdb.go b/providers/aws/documentdb.go index 9fd82ab..9a14241 100644 --- a/providers/aws/documentdb.go +++ b/providers/aws/documentdb.go @@ -19,7 +19,7 @@ type documentDBCluster struct { TTL int64 } -func listTaggedClusters(svc rds.RDS, tagName string) ([]documentDBCluster, error) { +func listTaggedDocumentDBClusters(svc rds.RDS, tagName string) ([]documentDBCluster, error) { var taggedClusters []documentDBCluster var instances []string @@ -69,7 +69,7 @@ func listTaggedClusters(svc rds.RDS, tagName string) ([]documentDBCluster, error return taggedClusters, nil } -func deleteCluster(svc rds.RDS, cluster documentDBCluster, dryRun bool) error { +func deleteDocumentDBCluster(svc rds.RDS, cluster documentDBCluster, dryRun bool) error { deleteInstancesErrors := 0 if cluster.Status == "deleting" { @@ -90,7 +90,7 @@ func deleteCluster(svc rds.RDS, cluster documentDBCluster, dryRun bool) error { continue } - err = deleteDatabase(svc, rdsInstanceInfo, dryRun) + err = deleteRDSDatabase(svc, rdsInstanceInfo, dryRun) if err != nil { log.Errorf("Deletion error on DocumentDB instance %s/%s/%s: %s", instance, cluster.DBClusterIdentifier, *svc.Config.Region, err) @@ -121,15 +121,15 @@ func deleteCluster(svc rds.RDS, cluster documentDBCluster, dryRun bool) error { return nil } -func DeleteExpiredClusters(svc rds.RDS, tagName string, dryRun bool) error { - clusters, err := listTaggedClusters(svc, tagName) +func DeleteExpiredDocumentDBClusters(svc rds.RDS, tagName string, dryRun bool) error { + clusters, err := listTaggedDocumentDBClusters(svc, tagName) if err != nil { return errors.New(fmt.Sprintf("can't list DocumentDB databases: %s\n", err)) } for _, cluster := range clusters { if utils.CheckIfExpired(cluster.ClusterCreateTime, cluster.TTL) { - err := deleteCluster(svc, cluster, dryRun) + err := deleteDocumentDBCluster(svc, cluster, dryRun) if err != nil { log.Errorf("Deletion DocumentDB cluster error %s/%s: %s", cluster.DBClusterIdentifier, *svc.Config.Region, err) diff --git a/providers/aws/elasticache.go b/providers/aws/elasticache.go new file mode 100644 index 0000000..c5d22ac --- /dev/null +++ b/providers/aws/elasticache.go @@ -0,0 +1,148 @@ +package aws + +import ( + "errors" + "fmt" + "github.com/Qovery/pleco/utils" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/elasticache" + log "github.com/sirupsen/logrus" + "strconv" + "time" +) + +type elasticacheCluster struct { + ClusterIdentifier string + ReplicationGroupId string + ClusterCreateTime time.Time + ClusterStatus string + TTL int64 +} + +func ElasticacheSession(sess session.Session, region string) *elasticache.ElastiCache { + return elasticache.New(&sess, &aws.Config{Region: aws.String(region)}) +} + +func listTaggedElasticacheDatabases(svc elasticache.ElastiCache, tagName string) ([]elasticacheCluster, error) { + var taggedClusters []elasticacheCluster + + log.Debugf("Listing all Elasticache clusters") + result, err := svc.DescribeCacheClusters(nil) + if err != nil { + return nil, err + } + + if len(result.CacheClusters) == 0 { + log.Debug("No Elasticache clusters were found") + return nil, nil + } + + for _, cluster := range result.CacheClusters { + tags, err := svc.ListTagsForResource( + &elasticache.ListTagsForResourceInput{ + ResourceName: aws.String(*cluster.ARN), + }, + ) + if err != nil { + if *cluster.CacheClusterStatus == "available" { + log.Errorf("Can't get tags for Elasticache cluster: %s", cluster.CacheClusterId) + } + continue + } + + for _, tag := range tags.TagList { + if *tag.Key == tagName { + if *tag.Key == "" { + log.Warn("Tag %s was empty and it wasn't expected, skipping", tag.Key) + continue + } + + ttl, err := strconv.Atoi(*tag.Value) + if err != nil { + log.Errorf("Error while trying to convert tag value (%s) to integer on instance %s in %s", + *tag.Value, *cluster.CacheClusterId, svc.Config.Region) + continue + } + + // required for replicas deletion + replicationGroupId := "" + if cluster.ReplicationGroupId != nil { + replicationGroupId = *cluster.ReplicationGroupId + } + + taggedClusters = append(taggedClusters, elasticacheCluster{ + ClusterIdentifier: *cluster.CacheClusterId, + ReplicationGroupId: replicationGroupId, + ClusterCreateTime: *cluster.CacheClusterCreateTime, + ClusterStatus: *cluster.CacheClusterStatus, + TTL: int64(ttl), + }) + } + } + } + log.Debugf("Found %d Elasticache cluster(s) in ready status with ttl tag", len(taggedClusters)) + + return taggedClusters, nil +} + +func deleteElasticacheCluster(svc elasticache.ElastiCache, cluster elasticacheCluster, dryRun bool) error { + if cluster.ClusterStatus == "deleting" { + log.Infof("Elasticache cluster %s is already in deletion process, skipping...", cluster.ClusterIdentifier) + return nil + } else { + log.Infof("Deleting Elasticache cluster %s in %s, expired after %d seconds", + cluster.ClusterIdentifier, *svc.Config.Region, cluster.TTL) + } + + if dryRun { + return nil + } + + // with replicas + if cluster.ReplicationGroupId != "" { + _, err := svc.DeleteReplicationGroup( + &elasticache.DeleteReplicationGroupInput{ + ReplicationGroupId: aws.String(cluster.ReplicationGroupId), + RetainPrimaryCluster: aws.Bool(false), + }, + ) + if err != nil { + return err + } + } + + _, err := svc.DeleteCacheCluster( + &elasticache.DeleteCacheClusterInput{ + CacheClusterId: aws.String(cluster.ClusterIdentifier), + }, + ) + if err != nil { + return err + } + + return nil +} + +func DeleteExpiredElasticacheDatabases(svc elasticache.ElastiCache, tagName string, dryRun bool) error { + clusters, err := listTaggedElasticacheDatabases(svc, tagName) + if err != nil { + return errors.New(fmt.Sprintf("can't list Elasticache databases: %s\n", err)) + } + + for _, cluster := range clusters { + if utils.CheckIfExpired(cluster.ClusterCreateTime, cluster.TTL) { + err := deleteElasticacheCluster(svc, cluster, dryRun) + if err != nil { + log.Errorf("Deletion Elasticache cluster error %s/%s: %s", + cluster.ClusterIdentifier, *svc.Config.Region, err) + continue + } + } else { + log.Debugf("Elasticache cluster %s in %s, has not yet expired", + cluster.ClusterIdentifier, *svc.Config.Region) + } + } + + return nil +} \ No newline at end of file diff --git a/providers/aws/rds.go b/providers/aws/rds.go index 0fb8830..0bf0e5c 100644 --- a/providers/aws/rds.go +++ b/providers/aws/rds.go @@ -23,7 +23,7 @@ func RdsSession(sess session.Session, region string) *rds.RDS { return rds.New(&sess, &aws.Config{Region: aws.String(region)}) } -func listTaggedDatabases(svc rds.RDS, tagName string) ([]rdsDatabase, error) { +func listTaggedRDSDatabases(svc rds.RDS, tagName string) ([]rdsDatabase, error) { var taggedDatabases []rdsDatabase log.Debugf("Listing all RDS databases") @@ -72,7 +72,7 @@ func listTaggedDatabases(svc rds.RDS, tagName string) ([]rdsDatabase, error) { return taggedDatabases, nil } -func deleteDatabase(svc rds.RDS, database rdsDatabase, dryRun bool) error { +func deleteRDSDatabase(svc rds.RDS, database rdsDatabase, dryRun bool) error { if database.DBInstanceStatus == "deleting" { log.Infof("RDS instance %s is already in deletion process, skipping...", database.DBInstanceIdentifier) return nil @@ -122,15 +122,15 @@ func getRDSInstanceInfos(svc rds.RDS, databaseIdentifier string) (rdsDatabase, e }, nil } -func DeleteExpiredDatabases(svc rds.RDS, tagName string, dryRun bool) error { - databases, err := listTaggedDatabases(svc, tagName) +func DeleteExpiredRDSDatabases(svc rds.RDS, tagName string, dryRun bool) error { + databases, err := listTaggedRDSDatabases(svc, tagName) if err != nil { return errors.New(fmt.Sprintf("can't list RDS databases: %s\n", err)) } for _, database := range databases { if utils.CheckIfExpired(database.InstanceCreateTime, database.TTL) { - err := deleteDatabase(svc, database, dryRun) + err := deleteRDSDatabase(svc, database, dryRun) if err != nil { log.Errorf("Deletion RDS database error %s/%s: %s", database.DBInstanceIdentifier, *svc.Config.Region, err)