From 8daf59aeb0e077c17dfa3bb64972ee025d880d3b Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Fri, 28 May 2021 14:10:01 +0200 Subject: [PATCH 01/68] #SOS-15 /collector/structs.go + AccountSpecifiedFiels struct to save AccountId and AccountName /collector/aws/common/structs.go + add Method getAccountName() /collector/aws/detector.go + save in DetectorManager the accountName from config file /collector/resources/* + save AccountSpecifiedFields with values in detectedStructs --- collector/aws/common/structs.go | 1 + collector/aws/detector.go | 17 ++++++++++++++++- collector/aws/resources/apigateway.go | 5 +++++ collector/aws/resources/docdb.go | 5 +++++ collector/aws/resources/dynamodb.go | 5 +++++ collector/aws/resources/ec2.go | 5 +++++ collector/aws/resources/ec2volumes.go | 5 +++++ collector/aws/resources/elasticache.go | 5 +++++ collector/aws/resources/elasticips.go | 5 +++++ collector/aws/resources/elasticsearch.go | 5 +++++ collector/aws/resources/elb.go | 5 +++++ collector/aws/resources/elbv2.go | 5 +++++ collector/aws/resources/iam.go | 5 +++++ collector/aws/resources/kinesis.go | 5 +++++ collector/aws/resources/lambda.go | 5 +++++ collector/aws/resources/natgateway.go | 5 +++++ collector/aws/resources/neptune.go | 5 +++++ collector/aws/resources/rds.go | 5 +++++ collector/aws/resources/redshift.go | 5 +++++ collector/structs.go | 6 ++++++ 20 files changed, 108 insertions(+), 1 deletion(-) diff --git a/collector/aws/common/structs.go b/collector/aws/common/structs.go index ebd7a22b..cb3241c5 100644 --- a/collector/aws/common/structs.go +++ b/collector/aws/common/structs.go @@ -26,6 +26,7 @@ type AWSManager interface { GetCloudWatchClient() *cloudwatch.CloudwatchManager GetPricingClient() *pricing.PricingManager GetRegion() string + GetAccountName() string GetSession() (*session.Session, *aws.Config) GetAccountIdentity() *sts.GetCallerIdentityOutput SetGlobal(resourceName collector.ResourceIdentifier) diff --git a/collector/aws/detector.go b/collector/aws/detector.go index 0450dfa9..857ae7f6 100644 --- a/collector/aws/detector.go +++ b/collector/aws/detector.go @@ -21,6 +21,7 @@ type DetectorDescriptor interface { GetCloudWatchClient() *cloudwatch.CloudwatchManager GetPricingClient() *pricing.PricingManager GetRegion() string + GetAccountName() string GetSession() (*session.Session, *awsClient.Config) GetAccountIdentity() *sts.GetCallerIdentityOutput } @@ -38,12 +39,20 @@ type DetectorManager struct { session *session.Session awsConfig *awsClient.Config accountIdentity *sts.GetCallerIdentityOutput + accountName string region string global map[string]struct{} } // NewDetectorManager create new instance of detector manager -func NewDetectorManager(awsAuth AuthDescriptor, collector collector.CollectorDescriber, account config.AWSAccount, stsManager *STSManager, global map[string]struct{}, region string) *DetectorManager { +func NewDetectorManager( + awsAuth AuthDescriptor, + collector collector.CollectorDescriber, + account config.AWSAccount, + stsManager *STSManager, + global map[string]struct{}, + region string, +) *DetectorManager { priceSession, _ := awsAuth.Login(defaultRegionPrice) pricingManager := pricing.NewPricingManager(awsPricing.New(priceSession), defaultRegionPrice) @@ -60,6 +69,7 @@ func NewDetectorManager(awsAuth AuthDescriptor, collector collector.CollectorDes session: regionSession, awsConfig: regionConfig, accountIdentity: callerIdentityOutput, + accountName: account.Name, global: global, } } @@ -89,6 +99,11 @@ func (dm *DetectorManager) GetRegion() string { return dm.region } +// GetAccountName returns the account name +func (dm *DetectorManager) GetAccountName() string { + return dm.accountName +} + // GetSession return the aws session func (dm *DetectorManager) GetSession() (*session.Session, *awsClient.Config) { return dm.session, dm.awsConfig diff --git a/collector/aws/resources/apigateway.go b/collector/aws/resources/apigateway.go index 0157e07d..e4fc5af2 100644 --- a/collector/aws/resources/apigateway.go +++ b/collector/aws/resources/apigateway.go @@ -36,6 +36,7 @@ type DetectedAPIGateway struct { Name string LaunchTime time.Time Tag map[string]string + collector.AccountSpecifiedFields } func init() { @@ -147,6 +148,10 @@ func (ag *APIGatewayManager) Detect(metrics []config.MetricConfig) (interface{}, Name: *api.Name, LaunchTime: *api.CreatedDate, Tag: tagsData, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ag.awsManager.GetAccountIdentity().Account, + AccountName: ag.awsManager.GetAccountName(), + }, } ag.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/docdb.go b/collector/aws/resources/docdb.go index 5753c94f..f6caabec 100644 --- a/collector/aws/resources/docdb.go +++ b/collector/aws/resources/docdb.go @@ -39,6 +39,7 @@ type DetectedDocumentDB struct { MultiAZ bool Engine string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -162,6 +163,10 @@ func (dd *DocumentDBManager) Detect(metrics []config.MetricConfig) (interface{}, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }, } dd.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/dynamodb.go b/collector/aws/resources/dynamodb.go index 6c109980..8b08b706 100644 --- a/collector/aws/resources/dynamodb.go +++ b/collector/aws/resources/dynamodb.go @@ -42,6 +42,7 @@ type DetectedAWSDynamoDB struct { Metric string Name string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -199,6 +200,10 @@ func (dd *DynamoDBManager) Detect(metrics []config.MetricConfig) (interface{}, e PricePerMonth: pricePerMonth, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }, } dd.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/ec2.go b/collector/aws/resources/ec2.go index 1bba1697..46663bbc 100644 --- a/collector/aws/resources/ec2.go +++ b/collector/aws/resources/ec2.go @@ -38,6 +38,7 @@ type DetectedEC2 struct { Name string InstanceType string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -164,6 +165,10 @@ func (ec *EC2Manager) Detect(metrics []config.MetricConfig) (interface{}, error) PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }, } ec.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/ec2volumes.go b/collector/aws/resources/ec2volumes.go index ac5e0fa3..a6db7710 100644 --- a/collector/aws/resources/ec2volumes.go +++ b/collector/aws/resources/ec2volumes.go @@ -36,6 +36,7 @@ type DetectedAWSEC2Volume struct { Size int64 PricePerMonth float64 Tag map[string]string + collector.AccountSpecifiedFields } func init() { @@ -124,6 +125,10 @@ func (ev *EC2VolumeManager) Detect(metrics []config.MetricConfig) (interface{}, Size: volumeSize, PricePerMonth: ev.getCalculatedPrice(vol, price), Tag: tagsData, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ev.awsManager.GetAccountIdentity().Account, + AccountName: ev.awsManager.GetAccountName(), + }, } ev.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/elasticache.go b/collector/aws/resources/elasticache.go index 473432ab..f0afc540 100644 --- a/collector/aws/resources/elasticache.go +++ b/collector/aws/resources/elasticache.go @@ -39,6 +39,7 @@ type DetectedElasticache struct { CacheNodeType string CacheNodes int collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -163,6 +164,10 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }, } ec.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/elasticips.go b/collector/aws/resources/elasticips.go index b22b94fd..ba43edef 100644 --- a/collector/aws/resources/elasticips.go +++ b/collector/aws/resources/elasticips.go @@ -35,6 +35,7 @@ type DetectedElasticIP struct { PricePerHour float64 PricePerMonth float64 Tag map[string]string + collector.AccountSpecifiedFields } func init() { @@ -115,6 +116,10 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ei.awsManager.GetAccountIdentity().Account, + AccountName: ei.awsManager.GetAccountName(), + }, } ei.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/elasticsearch.go b/collector/aws/resources/elasticsearch.go index 03e4e797..433beeb1 100644 --- a/collector/aws/resources/elasticsearch.go +++ b/collector/aws/resources/elasticsearch.go @@ -46,6 +46,7 @@ type DetectedElasticSearch struct { InstanceType string InstanceCount int64 collector.PriceDetectedFields + collector.AccountSpecifiedFields } // elasticSearchVolumeType will hold the available volume types for ESCluster EBS @@ -220,6 +221,10 @@ func (esm *ElasticSearchManager) Detect(metrics []config.MetricConfig) (interfac PricePerMonth: hourlyClusterPrice * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *esm.awsManager.GetAccountIdentity().Account, + AccountName: esm.awsManager.GetAccountName(), + }, } esm.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/elb.go b/collector/aws/resources/elb.go index 4ef6cfec..11a9594e 100644 --- a/collector/aws/resources/elb.go +++ b/collector/aws/resources/elb.go @@ -37,6 +37,7 @@ type DetectedELB struct { Metric string Region string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -176,6 +177,10 @@ func (el *ELBManager) Detect(metrics []config.MetricConfig) (interface{}, error) PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }, } el.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/elbv2.go b/collector/aws/resources/elbv2.go index c74ed42e..906a272c 100644 --- a/collector/aws/resources/elbv2.go +++ b/collector/aws/resources/elbv2.go @@ -38,6 +38,7 @@ type DetectedELBV2 struct { Region string Type string collector.PriceDetectedFields + collector.AccountSpecifiedFields } // loadBalancerConfig defines loadbalancer's configuration of metrics and pricing @@ -219,6 +220,10 @@ func (el *ELBV2Manager) Detect(metrics []config.MetricConfig) (interface{}, erro PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }, } el.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/iam.go b/collector/aws/resources/iam.go index 4aa8ca20..e2845024 100644 --- a/collector/aws/resources/iam.go +++ b/collector/aws/resources/iam.go @@ -34,6 +34,7 @@ type DetectedAWSLastActivity struct { AccessKey string LastUsedDate time.Time LastActivity string + collector.AccountSpecifiedFields } func init() { @@ -133,6 +134,10 @@ func (im *IAMManager) Detect(metrics []config.MetricConfig) (interface{}, error) AccessKey: *accessKeyData.AccessKeyId, LastUsedDate: lastUsedDate, LastActivity: lastActivity, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *im.awsManager.GetAccountIdentity().Account, + AccountName: im.awsManager.GetAccountName(), + }, } im.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index f1cd3d1c..6f30c255 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -38,6 +38,7 @@ type DetectedKinesis struct { Metric string Region string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -193,6 +194,10 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er PricePerMonth: totalShardsPerHourPrice * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *km.awsManager.GetAccountIdentity().Account, + AccountName: km.awsManager.GetAccountName(), + }, } km.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/lambda.go b/collector/aws/resources/lambda.go index e6e2569f..e22f96e5 100644 --- a/collector/aws/resources/lambda.go +++ b/collector/aws/resources/lambda.go @@ -36,6 +36,7 @@ type DetectedAWSLambda struct { ResourceID string Name string Tag map[string]string + collector.AccountSpecifiedFields } func init() { @@ -149,6 +150,10 @@ func (lm *LambdaManager) Detect(metrics []config.MetricConfig) (interface{}, err ResourceID: *fun.FunctionArn, Name: *fun.FunctionName, Tag: tagsData, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *lm.awsManager.GetAccountIdentity().Account, + AccountName: lm.awsManager.GetAccountName(), + }, } lm.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/natgateway.go b/collector/aws/resources/natgateway.go index a4f788e5..c15d1fc8 100644 --- a/collector/aws/resources/natgateway.go +++ b/collector/aws/resources/natgateway.go @@ -39,6 +39,7 @@ type DetectedNATGateway struct { SubnetID string VPCID string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -178,6 +179,10 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ngw.awsManager.GetAccountIdentity().Account, + AccountName: ngw.awsManager.GetAccountName(), + }, } ngw.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/neptune.go b/collector/aws/resources/neptune.go index 7a22ad3f..7092eee0 100644 --- a/collector/aws/resources/neptune.go +++ b/collector/aws/resources/neptune.go @@ -39,6 +39,7 @@ type DetectedAWSNeptune struct { MultiAZ bool Engine string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -163,6 +164,10 @@ func (np *NeptuneManager) Detect(metrics []config.MetricConfig) (interface{}, er PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *np.awsManager.GetAccountIdentity().Account, + AccountName: np.awsManager.GetAccountName(), + }, } np.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/rds.go b/collector/aws/resources/rds.go index ba2d5d67..6c9ee1a4 100644 --- a/collector/aws/resources/rds.go +++ b/collector/aws/resources/rds.go @@ -43,6 +43,7 @@ type DetectedAWSRDS struct { MultiAZ bool Engine string collector.PriceDetectedFields + collector.AccountSpecifiedFields } // RDSVolumeType will hold the available volume types for RDS types @@ -206,6 +207,10 @@ func (r *RDSManager) Detect(metrics []config.MetricConfig) (interface{}, error) PricePerMonth: totalHourlyPrice * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *r.awsManager.GetAccountIdentity().Account, + AccountName: r.awsManager.GetAccountName(), + }, } r.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/aws/resources/redshift.go b/collector/aws/resources/redshift.go index 7f660d6e..dc709e52 100644 --- a/collector/aws/resources/redshift.go +++ b/collector/aws/resources/redshift.go @@ -38,6 +38,7 @@ type DetectedRedShift struct { NodeType string NumberOfNodes int64 collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -159,6 +160,10 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, PricePerMonth: clusterPrice * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *rdm.awsManager.GetAccountIdentity().Account, + AccountName: rdm.awsManager.GetAccountName(), + }, } rdm.awsManager.GetCollector().AddResource(collector.EventCollector{ diff --git a/collector/structs.go b/collector/structs.go index 6ecd26f4..58119e3b 100644 --- a/collector/structs.go +++ b/collector/structs.go @@ -36,6 +36,12 @@ type PriceDetectedFields struct { Tag map[string]string } +// AccountSpecifiedFields describe account data of an resource +type AccountSpecifiedFields struct { + AccountID string + AccountName string +} + // EventCollector collector event data structure type EventCollector struct { EventType string From 35a3f71bac52f904b61e69e29a706a6d55e6c485 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Sat, 29 May 2021 15:53:57 +0200 Subject: [PATCH 02/68] #SOS-22 * elasticsearch.go getDynamicMatchQuery add: if filterName is Data.AccountID then build boolQuery for given AccountIDs --- api/storage/elasticsearch/elasticsearch.go | 25 ++++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/api/storage/elasticsearch/elasticsearch.go b/api/storage/elasticsearch/elasticsearch.go index afda70a2..7fee0744 100644 --- a/api/storage/elasticsearch/elasticsearch.go +++ b/api/storage/elasticsearch/elasticsearch.go @@ -10,6 +10,7 @@ import ( "fmt" "reflect" "strconv" + "strings" "time" elastic "github.com/olivere/elastic/v7" @@ -110,14 +111,24 @@ func (sm *StorageManager) getDynamicMatchQuery(filters map[string]string, operat dynamicMatchQuery := []elastic.Query{} var mq *elastic.MatchQuery for name, value := range filters { - mq = elastic.NewMatchQuery(name, value) - // Minimum number of clauses that must match for a document to be returned - mq.MinimumShouldMatch("100%") - if operator == "and" { - mq = mq.Operator("and") + if name == "Data.AccountID" { + var accountIds = strings.Split(value, ",") + var accountBoolQuery = elastic.NewBoolQuery() + for _, accountId := range accountIds { + accountBoolQuery.Should(elastic.NewMatchQuery(name, accountId)) + } + accountBoolQuery.MinimumShouldMatch("1") + dynamicMatchQuery = append(dynamicMatchQuery, accountBoolQuery) + } else { + mq = elastic.NewMatchQuery(name, value) + // Minimum number of clauses that must match for a document to be returned + mq.MinimumShouldMatch("100%") + if operator == "and" { + mq = mq.Operator("and") + } + log.Info("Query ", mq) + dynamicMatchQuery = append(dynamicMatchQuery, mq) } - - dynamicMatchQuery = append(dynamicMatchQuery, mq) } return dynamicMatchQuery } From 5866b6e615d9f3a20d7cabf8a6175e39182870ed Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Mon, 31 May 2021 15:05:44 +0200 Subject: [PATCH 03/68] #SOS-15 fit MockAWSManager to the modified AWSManager --- collector/aws/testutils/detector.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/collector/aws/testutils/detector.go b/collector/aws/testutils/detector.go index 4bab9eca..473e9385 100644 --- a/collector/aws/testutils/detector.go +++ b/collector/aws/testutils/detector.go @@ -17,11 +17,17 @@ type MockAWSManager struct { pricing *pricing.PricingManager session *session.Session accountIdentity *sts.GetCallerIdentityOutput + accountName string region string global map[string]struct{} } -func AWSManager(collector collector.CollectorDescriber, cloudWatchClient *cloudwatch.CloudwatchManager, priceClient *pricing.PricingManager, region string) *MockAWSManager { +func AWSManager( + collector collector.CollectorDescriber, + cloudWatchClient *cloudwatch.CloudwatchManager, + priceClient *pricing.PricingManager, + region string, +) *MockAWSManager { accountID := "1234" accountIdentity := &sts.GetCallerIdentityOutput{ @@ -34,6 +40,7 @@ func AWSManager(collector collector.CollectorDescriber, cloudWatchClient *cloudw pricing: priceClient, accountIdentity: accountIdentity, region: region, + accountName: "test", global: make(map[string]struct{}), } } @@ -58,6 +65,10 @@ func (dm *MockAWSManager) GetRegion() string { return dm.region } +func (dm *MockAWSManager) GetAccountName() string { + return dm.accountName +} + func (dm *MockAWSManager) GetSession() (*session.Session, *aws.Config) { return dm.session, &aws.Config{} } From 839ae2febccf74e4f9a601bacd8e377dc74e9178 Mon Sep 17 00:00:00 2001 From: daniebrill Date: Fri, 4 Jun 2021 20:22:31 +0200 Subject: [PATCH 04/68] #SOS-23 Add getAccounts + save account information in event_status --- api/route.go | 11 ++++++++ api/server.go | 1 + api/storage/elasticsearch/elasticsearch.go | 32 ++++++++++++++++++++++ api/storage/structs.go | 6 ++++ collector/aws/resources/apigateway.go | 10 +++++-- collector/aws/resources/docdb.go | 10 +++++-- collector/aws/resources/dynamodb.go | 10 +++++-- collector/aws/resources/ec2.go | 10 +++++-- collector/aws/resources/ec2volumes.go | 10 +++++-- collector/aws/resources/elasticache.go | 10 +++++-- collector/aws/resources/elasticips.go | 10 +++++-- collector/aws/resources/elasticsearch.go | 10 +++++-- collector/aws/resources/elb.go | 10 +++++-- collector/aws/resources/elbv2.go | 10 +++++-- collector/aws/resources/iam.go | 10 +++++-- collector/aws/resources/kinesis.go | 10 +++++-- collector/aws/resources/lambda.go | 10 +++++-- collector/aws/resources/natgateway.go | 10 +++++-- collector/aws/resources/neptune.go | 10 +++++-- collector/aws/resources/rds.go | 10 +++++-- collector/aws/resources/redshift.go | 10 +++++-- collector/collector.go | 14 ++++++---- collector/structs.go | 5 ++-- interpolation/interpolation.go | 9 ++++++ 24 files changed, 206 insertions(+), 42 deletions(-) diff --git a/api/route.go b/api/route.go index 2085741a..73357bc1 100644 --- a/api/route.go +++ b/api/route.go @@ -55,6 +55,17 @@ func (server *Server) GetExecutions(resp http.ResponseWriter, req *http.Request) server.JSONWrite(resp, http.StatusOK, results) } +func (server *Server) GetAccounts(resp http.ResponseWriter, req *http.Request) { + queryLimit, _ := strconv.Atoi(httpparameters.QueryParamWithDefault(req, "querylimit", storage.GetExecutionsQueryLimit)) + accounts, err := server.storage.GetAccounts(queryLimit) + if err != nil { + server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) + return + + } + server.JSONWrite(resp, http.StatusOK, accounts) +} + // GetResourceData return resuts details by resource type func (server *Server) GetResourceData(resp http.ResponseWriter, req *http.Request) { queryParams := req.URL.Query() diff --git a/api/server.go b/api/server.go index b82d45b1..10ad2498 100644 --- a/api/server.go +++ b/api/server.go @@ -81,6 +81,7 @@ func (server *Server) BindEndpoints() { server.router.HandleFunc("/api/v1/summary/{executionID}", server.GetSummary).Methods("GET") server.router.HandleFunc("/api/v1/executions", server.GetExecutions).Methods("GET") + server.router.HandleFunc("/api/v1/accounts", server.GetAccounts).Methods(("GET")) server.router.HandleFunc("/api/v1/resources/{type}", server.GetResourceData).Methods("GET") server.router.HandleFunc("/api/v1/trends/{type}", server.GetResourceTrends).Methods("GET") server.router.HandleFunc("/api/v1/tags/{executionID}", server.GetExecutionTags).Methods("GET") diff --git a/api/storage/elasticsearch/elasticsearch.go b/api/storage/elasticsearch/elasticsearch.go index 7fee0744..9dcee359 100644 --- a/api/storage/elasticsearch/elasticsearch.go +++ b/api/storage/elasticsearch/elasticsearch.go @@ -309,6 +309,38 @@ func (sm *StorageManager) GetExecutions(queryLimit int) ([]storage.Executions, e return executions, nil } +func (sm *StorageManager) GetAccounts(querylimit int) ([]storage.Accounts, error) { + accounts := []storage.Accounts{} + + searchResult, err := sm.client.Search().Aggregation("Accounts", elastic.NewTermsAggregation().Field("Data.AccountInformation.keyword")). + Do(context.Background()) + + if err != nil { + log.WithError(err).Error("error when trying to get AccountIDs") + return accounts, ErrInvalidQuery + } + + resp, ok := searchResult.Aggregations.Terms("Accounts") + if !ok { + log.Error("accounts field term does not exist") + return accounts, ErrAggregationTermNotFound + } + + for _, accountsBucket := range resp.Buckets { + account := accountsBucket.Key.(string) + name, id, err := interpolation.ExtractAccountInformation(account) + if err != nil { + log.WithError(err).WithField("account", account).Error("could not extract account information") + continue + } + accounts = append(accounts, storage.Accounts{ + ID: id, + Name: name, + }) + } + return accounts, nil +} + // GetResources return resource data func (sm *StorageManager) GetResources(resourceType string, executionID string, filters map[string]string) ([]map[string]interface{}, error) { diff --git a/api/storage/structs.go b/api/storage/structs.go index 467b8168..4d211301 100644 --- a/api/storage/structs.go +++ b/api/storage/structs.go @@ -13,6 +13,7 @@ type StorageDescriber interface { Save(data string) bool GetSummary(executionID string, filters map[string]string) (map[string]CollectorsSummary, error) GetExecutions(querylimit int) ([]Executions, error) + GetAccounts(querylimit int) ([]Accounts, error) GetResources(resourceType string, executionID string, filters map[string]string) ([]map[string]interface{}, error) GetResourceTrends(resourceType string, filters map[string]string, limit int) ([]ExecutionCost, error) GetExecutionTags(executionID string) (map[string][]string, error) @@ -31,6 +32,11 @@ type ExecutionCost struct { CostSum float64 } +type Accounts struct { + ID string + Name string +} + // CollectorsSummary defines unused resource summary type CollectorsSummary struct { ResourceName string `json:"ResourceName"` diff --git a/collector/aws/resources/apigateway.go b/collector/aws/resources/apigateway.go index e4fc5af2..118e5c65 100644 --- a/collector/aws/resources/apigateway.go +++ b/collector/aws/resources/apigateway.go @@ -72,7 +72,10 @@ func (ag *APIGatewayManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "apigateway", }).Info("starting to analyze resource") - ag.awsManager.GetCollector().CollectStart(ag.Name) + ag.awsManager.GetCollector().CollectStart(ag.Name, collector.AccountSpecifiedFields{ + AccountID: *ag.awsManager.GetAccountIdentity().Account, + AccountName: ag.awsManager.GetAccountName(), + }) detectAPIGateway := []DetectedAPIGateway{} apigateways, err := ag.getRestApis(nil, nil) @@ -165,7 +168,10 @@ func (ag *APIGatewayManager) Detect(metrics []config.MetricConfig) (interface{}, } } - ag.awsManager.GetCollector().CollectFinish(ag.Name) + ag.awsManager.GetCollector().CollectFinish(ag.Name, collector.AccountSpecifiedFields{ + AccountID: *ag.awsManager.GetAccountIdentity().Account, + AccountName: ag.awsManager.GetAccountName(), + }) return detectAPIGateway, nil } diff --git a/collector/aws/resources/docdb.go b/collector/aws/resources/docdb.go index f6caabec..1929dde3 100644 --- a/collector/aws/resources/docdb.go +++ b/collector/aws/resources/docdb.go @@ -75,7 +75,10 @@ func (dd *DocumentDBManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "documentDB", }).Info("starting to analyze resource") - dd.awsManager.GetCollector().CollectStart(dd.Name) + dd.awsManager.GetCollector().CollectStart(dd.Name, collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }) detectedDocDB := []DetectedDocumentDB{} instances, err := dd.describeInstances(nil, nil) @@ -181,7 +184,10 @@ func (dd *DocumentDBManager) Detect(metrics []config.MetricConfig) (interface{}, } - dd.awsManager.GetCollector().CollectFinish(dd.Name) + dd.awsManager.GetCollector().CollectFinish(dd.Name, collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }) return detectedDocDB, nil diff --git a/collector/aws/resources/dynamodb.go b/collector/aws/resources/dynamodb.go index 8b08b706..084b1260 100644 --- a/collector/aws/resources/dynamodb.go +++ b/collector/aws/resources/dynamodb.go @@ -79,7 +79,10 @@ func (dd *DynamoDBManager) Detect(metrics []config.MetricConfig) (interface{}, e "resource": "dynamoDB", }).Info("starting to analyze resource") - dd.awsManager.GetCollector().CollectStart(dd.Name) + dd.awsManager.GetCollector().CollectStart(dd.Name, collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }) detectedTables := []DetectedAWSDynamoDB{} tables, err := dd.describeTables(nil, nil) @@ -217,7 +220,10 @@ func (dd *DynamoDBManager) Detect(metrics []config.MetricConfig) (interface{}, e } } - dd.awsManager.GetCollector().CollectFinish(dd.Name) + dd.awsManager.GetCollector().CollectFinish(dd.Name, collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }) return detectedTables, nil diff --git a/collector/aws/resources/ec2.go b/collector/aws/resources/ec2.go index 46663bbc..52fe9f70 100644 --- a/collector/aws/resources/ec2.go +++ b/collector/aws/resources/ec2.go @@ -74,7 +74,10 @@ func (ec *EC2Manager) Detect(metrics []config.MetricConfig) (interface{}, error) "resource": "ec2_instances", }).Info("starting to analyze resource") - ec.awsManager.GetCollector().CollectStart(ec.Name) + ec.awsManager.GetCollector().CollectStart(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) detectedEC2 := []DetectedEC2{} @@ -183,7 +186,10 @@ func (ec *EC2Manager) Detect(metrics []config.MetricConfig) (interface{}, error) } } - ec.awsManager.GetCollector().CollectFinish(ec.Name) + ec.awsManager.GetCollector().CollectFinish(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) return detectedEC2, nil diff --git a/collector/aws/resources/ec2volumes.go b/collector/aws/resources/ec2volumes.go index a6db7710..b4a5ccc0 100644 --- a/collector/aws/resources/ec2volumes.go +++ b/collector/aws/resources/ec2volumes.go @@ -76,7 +76,10 @@ func (ev *EC2VolumeManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "ec2_volume", }).Info("starting to analyze resource") - ev.awsManager.GetCollector().CollectStart(ev.Name) + ev.awsManager.GetCollector().CollectStart(ev.Name, collector.AccountSpecifiedFields{ + AccountID: *ev.awsManager.GetAccountIdentity().Account, + AccountName: ev.awsManager.GetAccountName(), + }) detected := []DetectedAWSEC2Volume{} volumes, err := ev.describe(nil, nil) @@ -140,7 +143,10 @@ func (ev *EC2VolumeManager) Detect(metrics []config.MetricConfig) (interface{}, } - ev.awsManager.GetCollector().CollectFinish(ev.Name) + ev.awsManager.GetCollector().CollectFinish(ev.Name, collector.AccountSpecifiedFields{ + AccountID: *ev.awsManager.GetAccountIdentity().Account, + AccountName: ev.awsManager.GetAccountName(), + }) return detected, nil diff --git a/collector/aws/resources/elasticache.go b/collector/aws/resources/elasticache.go index f0afc540..96c9c4c7 100644 --- a/collector/aws/resources/elasticache.go +++ b/collector/aws/resources/elasticache.go @@ -75,7 +75,10 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} "resource": "elasticache", }).Info("starting to analyze resource") - ec.awsManager.GetCollector().CollectStart(ec.Name) + ec.awsManager.GetCollector().CollectStart(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) detectedelasticache := []DetectedElasticache{} @@ -180,7 +183,10 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} } } - ec.awsManager.GetCollector().CollectFinish(ec.Name) + ec.awsManager.GetCollector().CollectFinish(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) return detectedelasticache, nil } diff --git a/collector/aws/resources/elasticips.go b/collector/aws/resources/elasticips.go index ba43edef..27336e3c 100644 --- a/collector/aws/resources/elasticips.go +++ b/collector/aws/resources/elasticips.go @@ -73,7 +73,10 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "elastic ips", }).Info("starting to analyze resource") - ei.awsManager.GetCollector().CollectStart(ei.Name) + ei.awsManager.GetCollector().CollectStart(ei.Name, collector.AccountSpecifiedFields{ + AccountID: *ei.awsManager.GetAccountIdentity().Account, + AccountName: ei.awsManager.GetAccountName(), + }) elasticIPs := []DetectedElasticIP{} @@ -132,7 +135,10 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, } } - ei.awsManager.GetCollector().CollectFinish(ei.Name) + ei.awsManager.GetCollector().CollectFinish(ei.Name, collector.AccountSpecifiedFields{ + AccountID: *ei.awsManager.GetAccountIdentity().Account, + AccountName: ei.awsManager.GetAccountName(), + }) return elasticIPs, nil diff --git a/collector/aws/resources/elasticsearch.go b/collector/aws/resources/elasticsearch.go index 433beeb1..13a7fb38 100644 --- a/collector/aws/resources/elasticsearch.go +++ b/collector/aws/resources/elasticsearch.go @@ -89,7 +89,10 @@ func (esm *ElasticSearchManager) Detect(metrics []config.MetricConfig) (interfac "resource": "elasticsearch", }).Info("analyzing resource") - esm.awsManager.GetCollector().CollectStart(esm.Name) + esm.awsManager.GetCollector().CollectStart(esm.Name, collector.AccountSpecifiedFields{ + AccountID: *esm.awsManager.GetAccountIdentity().Account, + AccountName: esm.awsManager.GetAccountName(), + }) detectedElasticSearchClusters := []DetectedElasticSearch{} @@ -237,7 +240,10 @@ func (esm *ElasticSearchManager) Detect(metrics []config.MetricConfig) (interfac } } - esm.awsManager.GetCollector().CollectFinish(esm.Name) + esm.awsManager.GetCollector().CollectFinish(esm.Name, collector.AccountSpecifiedFields{ + AccountID: *esm.awsManager.GetAccountIdentity().Account, + AccountName: esm.awsManager.GetAccountName(), + }) return detectedElasticSearchClusters, nil } diff --git a/collector/aws/resources/elb.go b/collector/aws/resources/elb.go index 11a9594e..14b77d1c 100644 --- a/collector/aws/resources/elb.go +++ b/collector/aws/resources/elb.go @@ -73,7 +73,10 @@ func (el *ELBManager) Detect(metrics []config.MetricConfig) (interface{}, error) "resource": "elb", }).Info("starting to analyze resource") - el.awsManager.GetCollector().CollectStart(el.Name) + el.awsManager.GetCollector().CollectStart(el.Name, collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }) detectedELB := []DetectedELB{} @@ -195,7 +198,10 @@ func (el *ELBManager) Detect(metrics []config.MetricConfig) (interface{}, error) } } - el.awsManager.GetCollector().CollectFinish(el.Name) + el.awsManager.GetCollector().CollectFinish(el.Name, collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }) return detectedELB, nil diff --git a/collector/aws/resources/elbv2.go b/collector/aws/resources/elbv2.go index 906a272c..4bc5e2b4 100644 --- a/collector/aws/resources/elbv2.go +++ b/collector/aws/resources/elbv2.go @@ -104,7 +104,10 @@ func (el *ELBV2Manager) Detect(metrics []config.MetricConfig) (interface{}, erro "resource": "elb_v2", }).Info("starting to analyze resource") - el.awsManager.GetCollector().CollectStart(el.Name) + el.awsManager.GetCollector().CollectStart(el.Name, collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }) detectedELBV2 := []DetectedELBV2{} @@ -236,7 +239,10 @@ func (el *ELBV2Manager) Detect(metrics []config.MetricConfig) (interface{}, erro } } - el.awsManager.GetCollector().CollectFinish(el.Name) + el.awsManager.GetCollector().CollectFinish(el.Name, collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }) return detectedELBV2, nil diff --git a/collector/aws/resources/iam.go b/collector/aws/resources/iam.go index e2845024..dcd12e66 100644 --- a/collector/aws/resources/iam.go +++ b/collector/aws/resources/iam.go @@ -76,7 +76,10 @@ func (im *IAMManager) Detect(metrics []config.MetricConfig) (interface{}, error) "resource": "iam", }).Info("starting to analyze resource") - im.awsManager.GetCollector().CollectStart(im.Name) + im.awsManager.GetCollector().CollectStart(im.Name, collector.AccountSpecifiedFields{ + AccountID: *im.awsManager.GetAccountIdentity().Account, + AccountName: im.awsManager.GetAccountName(), + }) detected := []DetectedAWSLastActivity{} @@ -151,7 +154,10 @@ func (im *IAMManager) Detect(metrics []config.MetricConfig) (interface{}, error) } } - im.awsManager.GetCollector().CollectFinish(im.Name) + im.awsManager.GetCollector().CollectFinish(im.Name, collector.AccountSpecifiedFields{ + AccountID: *im.awsManager.GetAccountIdentity().Account, + AccountName: im.awsManager.GetAccountName(), + }) return detected, nil } diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index 6f30c255..b5471346 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -75,7 +75,10 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er "resource": "kinesis", }).Info("analyzing resource") - km.awsManager.GetCollector().CollectStart(km.Name) + km.awsManager.GetCollector().CollectStart(km.Name, collector.AccountSpecifiedFields{ + AccountID: *km.awsManager.GetAccountIdentity().Account, + AccountName: km.awsManager.GetAccountName(), + }) streams, err := km.describeStreams(nil, nil) if err != nil { @@ -209,7 +212,10 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er } } } - km.awsManager.GetCollector().CollectFinish(km.Name) + km.awsManager.GetCollector().CollectFinish(km.Name, collector.AccountSpecifiedFields{ + AccountID: *km.awsManager.GetAccountIdentity().Account, + AccountName: km.awsManager.GetAccountName(), + }) return detectedStreams, nil } diff --git a/collector/aws/resources/lambda.go b/collector/aws/resources/lambda.go index e22f96e5..4de76190 100644 --- a/collector/aws/resources/lambda.go +++ b/collector/aws/resources/lambda.go @@ -71,7 +71,10 @@ func (lm *LambdaManager) Detect(metrics []config.MetricConfig) (interface{}, err "resource": "lambda", }).Info("starting to analyze resource") - lm.awsManager.GetCollector().CollectStart(lm.Name) + lm.awsManager.GetCollector().CollectStart(lm.Name, collector.AccountSpecifiedFields{ + AccountID: *lm.awsManager.GetAccountIdentity().Account, + AccountName: lm.awsManager.GetAccountName(), + }) detected := []DetectedAWSLambda{} functions, err := lm.describe(nil, nil) @@ -166,7 +169,10 @@ func (lm *LambdaManager) Detect(metrics []config.MetricConfig) (interface{}, err } } - lm.awsManager.GetCollector().CollectFinish(lm.Name) + lm.awsManager.GetCollector().CollectFinish(lm.Name, collector.AccountSpecifiedFields{ + AccountID: *lm.awsManager.GetAccountIdentity().Account, + AccountName: lm.awsManager.GetAccountName(), + }) return detected, nil } diff --git a/collector/aws/resources/natgateway.go b/collector/aws/resources/natgateway.go index c15d1fc8..ded329f9 100644 --- a/collector/aws/resources/natgateway.go +++ b/collector/aws/resources/natgateway.go @@ -75,7 +75,10 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} "resource": "natgateway", }).Info("analyzing resource") - ngw.awsManager.GetCollector().CollectStart(ngw.Name) + ngw.awsManager.GetCollector().CollectStart(ngw.Name, collector.AccountSpecifiedFields{ + AccountID: *ngw.awsManager.GetAccountIdentity().Account, + AccountName: ngw.awsManager.GetAccountName(), + }) DetectedNATGateways := []DetectedNATGateway{} @@ -195,7 +198,10 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} } } - ngw.awsManager.GetCollector().CollectFinish(ngw.Name) + ngw.awsManager.GetCollector().CollectFinish(ngw.Name, collector.AccountSpecifiedFields{ + AccountID: *ngw.awsManager.GetAccountIdentity().Account, + AccountName: ngw.awsManager.GetAccountName(), + }) return DetectedNATGateways, nil } diff --git a/collector/aws/resources/neptune.go b/collector/aws/resources/neptune.go index 7092eee0..8a46eebc 100644 --- a/collector/aws/resources/neptune.go +++ b/collector/aws/resources/neptune.go @@ -75,7 +75,10 @@ func (np *NeptuneManager) Detect(metrics []config.MetricConfig) (interface{}, er "resource": "neptune", }).Info("starting to analyze resource") - np.awsManager.GetCollector().CollectStart(np.Name) + np.awsManager.GetCollector().CollectStart(np.Name, collector.AccountSpecifiedFields{ + AccountID: *np.awsManager.GetAccountIdentity().Account, + AccountName: np.awsManager.GetAccountName(), + }) detected := []DetectedAWSNeptune{} instances, err := np.describeInstances(nil, nil) @@ -178,7 +181,10 @@ func (np *NeptuneManager) Detect(metrics []config.MetricConfig) (interface{}, er } } } - np.awsManager.GetCollector().CollectFinish(np.Name) + np.awsManager.GetCollector().CollectFinish(np.Name, collector.AccountSpecifiedFields{ + AccountID: *np.awsManager.GetAccountIdentity().Account, + AccountName: np.awsManager.GetAccountName(), + }) return detected, nil } diff --git a/collector/aws/resources/rds.go b/collector/aws/resources/rds.go index 6c9ee1a4..179bebfa 100644 --- a/collector/aws/resources/rds.go +++ b/collector/aws/resources/rds.go @@ -87,7 +87,10 @@ func (r *RDSManager) Detect(metrics []config.MetricConfig) (interface{}, error) "resource": "rds", }).Info("starting to analyze resource") - r.awsManager.GetCollector().CollectStart(r.Name) + r.awsManager.GetCollector().CollectStart(r.Name, collector.AccountSpecifiedFields{ + AccountID: *r.awsManager.GetAccountIdentity().Account, + AccountName: r.awsManager.GetAccountName(), + }) detected := []DetectedAWSRDS{} @@ -224,7 +227,10 @@ func (r *RDSManager) Detect(metrics []config.MetricConfig) (interface{}, error) } - r.awsManager.GetCollector().CollectFinish(r.Name) + r.awsManager.GetCollector().CollectFinish(r.Name, collector.AccountSpecifiedFields{ + AccountID: *r.awsManager.GetAccountIdentity().Account, + AccountName: r.awsManager.GetAccountName(), + }) return detected, nil diff --git a/collector/aws/resources/redshift.go b/collector/aws/resources/redshift.go index dc709e52..4dc3d7e0 100644 --- a/collector/aws/resources/redshift.go +++ b/collector/aws/resources/redshift.go @@ -74,7 +74,10 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "redshift", }).Info("analyzing resource") - rdm.awsManager.GetCollector().CollectStart(rdm.Name) + rdm.awsManager.GetCollector().CollectStart(rdm.Name, collector.AccountSpecifiedFields{ + AccountID: *rdm.awsManager.GetAccountIdentity().Account, + AccountName: rdm.awsManager.GetAccountName(), + }) detectedredshiftClusters := []DetectedRedShift{} @@ -176,7 +179,10 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, } } - rdm.awsManager.GetCollector().CollectFinish(rdm.Name) + rdm.awsManager.GetCollector().CollectFinish(rdm.Name, collector.AccountSpecifiedFields{ + AccountID: *rdm.awsManager.GetAccountIdentity().Account, + AccountName: rdm.awsManager.GetAccountName(), + }) return detectedredshiftClusters, nil } diff --git a/collector/collector.go b/collector/collector.go index b53b48c5..f3631f42 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -25,8 +25,8 @@ const ( // CollectorDescriber describe the collector functions type CollectorDescriber interface { AddResource(data EventCollector) - CollectStart(resourceName ResourceIdentifier) - CollectFinish(resourceName ResourceIdentifier) + CollectStart(resourceName ResourceIdentifier, accountSpecifiedFields AccountSpecifiedFields) + CollectFinish(resourceName ResourceIdentifier, accountSpecifiedFields AccountSpecifiedFields) CollectError(resourceName ResourceIdentifier, err error) GetCollectorEvent() []EventCollector } @@ -98,21 +98,23 @@ func (cm *CollectorManager) AddResource(data EventCollector) { } // CollectStart add `fetch` event to collector by given resource name -func (cm *CollectorManager) CollectStart(resourceName ResourceIdentifier) { +func (cm *CollectorManager) CollectStart(resourceName ResourceIdentifier, accountSpecifiedFields AccountSpecifiedFields) { cm.updateServiceStatus(EventCollector{ ResourceName: resourceName, Data: EventStatusData{ - Status: EventFetch, + Status: EventFetch, + AccountInformation: accountSpecifiedFields.AccountName + "_" + accountSpecifiedFields.AccountID, }, }) } // CollectFinish add `finish` event to collector by given resource name -func (cm *CollectorManager) CollectFinish(resourceName ResourceIdentifier) { +func (cm *CollectorManager) CollectFinish(resourceName ResourceIdentifier, accountSpecifiedFields AccountSpecifiedFields) { cm.updateServiceStatus(EventCollector{ ResourceName: resourceName, Data: EventStatusData{ - Status: EventFinish, + Status: EventFinish, + AccountInformation: accountSpecifiedFields.AccountName + "_" + accountSpecifiedFields.AccountID, }, }) } diff --git a/collector/structs.go b/collector/structs.go index 58119e3b..9dc29506 100644 --- a/collector/structs.go +++ b/collector/structs.go @@ -23,8 +23,9 @@ const ( // EventStatusData descrive the struct of the resource statuses type EventStatusData struct { - Status EventStatus - ErrorMessage string + Status EventStatus + ErrorMessage string + AccountInformation string } // PriceDetectedFields describe the pricing field diff --git a/interpolation/interpolation.go b/interpolation/interpolation.go index f7268f17..8c936354 100644 --- a/interpolation/interpolation.go +++ b/interpolation/interpolation.go @@ -59,3 +59,12 @@ func ExtractExecutionName(executionId string) (string, error) { return executionArr[0], nil } + +// ExtractAccountInformation will return Account Name and Account ID +func ExtractAccountInformation(account string) (string, string, error) { + info := strings.Split(account, "_") + if len(info) != 2 { + return "", "", errors.New("unexpected account format") + } + return info[0], info[1], nil +} From 6cd0a85a4a6e893e54c73c3ff99c984ad34dbde3 Mon Sep 17 00:00:00 2001 From: daniebrill Date: Sat, 5 Jun 2021 19:11:34 +0200 Subject: [PATCH 05/68] #SOS-23 Use querylimit in aggregation + add error handling for type assertion --- api/storage/elasticsearch/elasticsearch.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/api/storage/elasticsearch/elasticsearch.go b/api/storage/elasticsearch/elasticsearch.go index 9dcee359..836840ed 100644 --- a/api/storage/elasticsearch/elasticsearch.go +++ b/api/storage/elasticsearch/elasticsearch.go @@ -312,7 +312,8 @@ func (sm *StorageManager) GetExecutions(queryLimit int) ([]storage.Executions, e func (sm *StorageManager) GetAccounts(querylimit int) ([]storage.Accounts, error) { accounts := []storage.Accounts{} - searchResult, err := sm.client.Search().Aggregation("Accounts", elastic.NewTermsAggregation().Field("Data.AccountInformation.keyword")). + searchResult, err := sm.client.Search().Aggregation("Accounts", elastic.NewTermsAggregation(). + Field("Data.AccountInformation.keyword").Size(querylimit)). Do(context.Background()) if err != nil { @@ -327,7 +328,11 @@ func (sm *StorageManager) GetAccounts(querylimit int) ([]storage.Accounts, error } for _, accountsBucket := range resp.Buckets { - account := accountsBucket.Key.(string) + account, ok := accountsBucket.Key.(string) + if !ok { + log.Error("type assertion to string failed") + continue + } name, id, err := interpolation.ExtractAccountInformation(account) if err != nil { log.WithError(err).WithField("account", account).Error("could not extract account information") From f8e931b57399d3da11fd686e7b7abccb24b3c4b9 Mon Sep 17 00:00:00 2001 From: daniebrill Date: Tue, 8 Jun 2021 13:53:06 +0200 Subject: [PATCH 06/68] #SOS-23 Fix existing tests + code format --- api/testutils/storage.go | 14 ++++++++++++++ collector/collector_test.go | 10 ++++++++-- collector/testutils/collector.go | 10 ++++++---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/api/testutils/storage.go b/api/testutils/storage.go index 7a1a36a2..5100943c 100644 --- a/api/testutils/storage.go +++ b/api/testutils/storage.go @@ -65,6 +65,20 @@ func (ms *MockStorage) GetExecutions(queryLimit int) ([]storage.Executions, erro return response, nil } +func (ms *MockStorage) GetAccounts(querylimit int) ([]storage.Accounts, error) { + response := []storage.Accounts{ + { + ID: "1234567890", + Name: "Test1", + }, + { + ID: "1234567891", + Name: "Test2", + }, + } + return response, nil +} + func (ms *MockStorage) GetResources(resourceType string, executionID string, filters map[string]string) ([]map[string]interface{}, error) { var response []map[string]interface{} diff --git a/collector/collector_test.go b/collector/collector_test.go index 55224438..88931e85 100644 --- a/collector/collector_test.go +++ b/collector/collector_test.go @@ -78,7 +78,10 @@ func TestAddEvent(t *testing.T) { time.Sleep(time.Second) - coll.CollectStart(collector.ResourceIdentifier("test")) + coll.CollectStart(collector.ResourceIdentifier("test"), collector.AccountSpecifiedFields{ + AccountName: "Test", + AccountID: "1234567890", + }) coll.AddResource(collector.EventCollector{ ResourceName: "test1", Data: "test data", @@ -122,7 +125,10 @@ func TestAddEventServerUnavailable(t *testing.T) { time.Sleep(time.Second) - coll.CollectStart(collector.ResourceIdentifier("test")) + coll.CollectStart(collector.ResourceIdentifier("test"), collector.AccountSpecifiedFields{ + AccountName: "Test", + AccountID: "1234567890", + }) coll.AddResource(collector.EventCollector{ ResourceName: "test1", diff --git a/collector/testutils/collector.go b/collector/testutils/collector.go index 0a41c9bb..81b7a9cc 100644 --- a/collector/testutils/collector.go +++ b/collector/testutils/collector.go @@ -23,20 +23,22 @@ func (mc *MockCollector) GetCollectorEvent() []collector.EventCollector { return events } -func (mc *MockCollector) CollectStart(resourceName collector.ResourceIdentifier) { +func (mc *MockCollector) CollectStart(resourceName collector.ResourceIdentifier, accountSpecifiedFields collector.AccountSpecifiedFields) { mc.updateServiceStatus(collector.EventCollector{ ResourceName: resourceName, Data: collector.EventStatusData{ - Status: collector.EventFetch, + Status: collector.EventFetch, + AccountInformation: accountSpecifiedFields.AccountName + "_" + accountSpecifiedFields.AccountID, }, }) } -func (mc *MockCollector) CollectFinish(resourceName collector.ResourceIdentifier) { +func (mc *MockCollector) CollectFinish(resourceName collector.ResourceIdentifier, accountSpecifiedFields collector.AccountSpecifiedFields) { mc.updateServiceStatus(collector.EventCollector{ ResourceName: resourceName, Data: collector.EventStatusData{ - Status: collector.EventFinish, + Status: collector.EventFinish, + AccountInformation: accountSpecifiedFields.AccountName + "_" + accountSpecifiedFields.AccountID, }, }) From 259ccf7368fac954a659e97640deedb801d9b83b Mon Sep 17 00:00:00 2001 From: daniebrill Date: Tue, 8 Jun 2021 15:14:22 +0200 Subject: [PATCH 07/68] #SOS-23 Modify GetAccounts for specific execution ID --- api/route.go | 5 ++++- api/server.go | 2 +- api/storage/elasticsearch/elasticsearch.go | 7 ++++--- api/storage/structs.go | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/api/route.go b/api/route.go index 73357bc1..756c2f32 100644 --- a/api/route.go +++ b/api/route.go @@ -57,7 +57,10 @@ func (server *Server) GetExecutions(resp http.ResponseWriter, req *http.Request) func (server *Server) GetAccounts(resp http.ResponseWriter, req *http.Request) { queryLimit, _ := strconv.Atoi(httpparameters.QueryParamWithDefault(req, "querylimit", storage.GetExecutionsQueryLimit)) - accounts, err := server.storage.GetAccounts(queryLimit) + params := mux.Vars(req) + executionID := params["executionID"] + accounts, err := server.storage.GetAccounts(executionID, queryLimit) + if err != nil { server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) return diff --git a/api/server.go b/api/server.go index 10ad2498..357c9c69 100644 --- a/api/server.go +++ b/api/server.go @@ -81,7 +81,7 @@ func (server *Server) BindEndpoints() { server.router.HandleFunc("/api/v1/summary/{executionID}", server.GetSummary).Methods("GET") server.router.HandleFunc("/api/v1/executions", server.GetExecutions).Methods("GET") - server.router.HandleFunc("/api/v1/accounts", server.GetAccounts).Methods(("GET")) + server.router.HandleFunc("/api/v1/accounts/{executionID}", server.GetAccounts).Methods(("GET")) server.router.HandleFunc("/api/v1/resources/{type}", server.GetResourceData).Methods("GET") server.router.HandleFunc("/api/v1/trends/{type}", server.GetResourceTrends).Methods("GET") server.router.HandleFunc("/api/v1/tags/{executionID}", server.GetExecutionTags).Methods("GET") diff --git a/api/storage/elasticsearch/elasticsearch.go b/api/storage/elasticsearch/elasticsearch.go index 836840ed..0fd4403b 100644 --- a/api/storage/elasticsearch/elasticsearch.go +++ b/api/storage/elasticsearch/elasticsearch.go @@ -309,11 +309,12 @@ func (sm *StorageManager) GetExecutions(queryLimit int) ([]storage.Executions, e return executions, nil } -func (sm *StorageManager) GetAccounts(querylimit int) ([]storage.Accounts, error) { +func (sm *StorageManager) GetAccounts(executionID string, querylimit int) ([]storage.Accounts, error) { accounts := []storage.Accounts{} - searchResult, err := sm.client.Search().Aggregation("Accounts", elastic.NewTermsAggregation(). - Field("Data.AccountInformation.keyword").Size(querylimit)). + searchResult, err := sm.client.Search().Query(elastic.NewMatchQuery("ExecutionID", executionID)). + Aggregation("Accounts", elastic.NewTermsAggregation(). + Field("Data.AccountInformation.keyword").Size(querylimit)). Do(context.Background()) if err != nil { diff --git a/api/storage/structs.go b/api/storage/structs.go index 4d211301..a2bef4d6 100644 --- a/api/storage/structs.go +++ b/api/storage/structs.go @@ -13,7 +13,7 @@ type StorageDescriber interface { Save(data string) bool GetSummary(executionID string, filters map[string]string) (map[string]CollectorsSummary, error) GetExecutions(querylimit int) ([]Executions, error) - GetAccounts(querylimit int) ([]Accounts, error) + GetAccounts(executionID string, querylimit int) ([]Accounts, error) GetResources(resourceType string, executionID string, filters map[string]string) ([]map[string]interface{}, error) GetResourceTrends(resourceType string, filters map[string]string, limit int) ([]ExecutionCost, error) GetExecutionTags(executionID string) (map[string][]string, error) From 45db9b4d32310cea23153764872712d5ffffb4bf Mon Sep 17 00:00:00 2001 From: daniebrill Date: Tue, 8 Jun 2021 15:30:04 +0200 Subject: [PATCH 08/68] #SOS-23 Fit MockStorage to modified interface StorageDescriber --- api/testutils/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/testutils/storage.go b/api/testutils/storage.go index 5100943c..7293503a 100644 --- a/api/testutils/storage.go +++ b/api/testutils/storage.go @@ -65,7 +65,7 @@ func (ms *MockStorage) GetExecutions(queryLimit int) ([]storage.Executions, erro return response, nil } -func (ms *MockStorage) GetAccounts(querylimit int) ([]storage.Accounts, error) { +func (ms *MockStorage) GetAccounts(executionID string, querylimit int) ([]storage.Accounts, error) { response := []storage.Accounts{ { ID: "1234567890", From 2f3c15cd9c083168ee35cdd2ef48eb2b1f2f2f04 Mon Sep 17 00:00:00 2001 From: daniebrill Date: Tue, 8 Jun 2021 18:31:03 +0200 Subject: [PATCH 09/68] #SOS-23 Add tests in interpolation_test.go and server_test.go --- api/server_test.go | 49 +++++++++++++++++++++++++++++ api/testutils/storage.go | 3 ++ interpolation/interpolation_test.go | 25 +++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/api/server_test.go b/api/server_test.go index 7ce3ccf0..a7cad068 100644 --- a/api/server_test.go +++ b/api/server_test.go @@ -235,8 +235,57 @@ func TestGetExecutions(t *testing.T) { }) } +} + +func TestGetAccounts(t *testing.T) { + ms, _ := MockServer() + ms.BindEndpoints() + ms.Serve() + testCases := []struct { + endpoint string + expectedStatusCode int + Count int + }{ + {"/api/v1/accounts", http.StatusNotFound, 0}, + {"/api/v1/accounts/1", http.StatusOK, 2}, + {"/api/v1/accounts/err", http.StatusInternalServerError, 0}, + } + + for _, test := range testCases { + t.Run(test.endpoint, func(t *testing.T) { + + rr := httptest.NewRecorder() + req, err := http.NewRequest("GET", test.endpoint, nil) + if err != nil { + t.Fatal(err) + } + ms.Router().ServeHTTP(rr, req) + if rr.Code != test.expectedStatusCode { + t.Fatalf("handler returned wrong status code: got %v want %v", rr.Code, test.expectedStatusCode) + } + + if test.expectedStatusCode == http.StatusOK { + body, err := ioutil.ReadAll(rr.Body) + if err != nil { + t.Fatal(err) + } + + var accountsData []storage.Accounts + + err = json.Unmarshal(body, &accountsData) + if err != nil { + t.Fatalf("Could not parse http response") + } + + if len(accountsData) != test.Count { + t.Fatalf("unexpected resources summary response, got %d expected %d", len(accountsData), test.Count) + } + } + }) + } } + func TestSave(t *testing.T) { ms, mockStorage := MockServer() ms.BindEndpoints() diff --git a/api/testutils/storage.go b/api/testutils/storage.go index 7293503a..cc3034b5 100644 --- a/api/testutils/storage.go +++ b/api/testutils/storage.go @@ -66,6 +66,9 @@ func (ms *MockStorage) GetExecutions(queryLimit int) ([]storage.Executions, erro } func (ms *MockStorage) GetAccounts(executionID string, querylimit int) ([]storage.Accounts, error) { + if executionID == "err" { + return nil, errors.New("error") + } response := []storage.Accounts{ { ID: "1234567890", diff --git a/interpolation/interpolation_test.go b/interpolation/interpolation_test.go index 51e5d4db..47612441 100644 --- a/interpolation/interpolation_test.go +++ b/interpolation/interpolation_test.go @@ -71,3 +71,28 @@ func TestExtractExecutionName(t *testing.T) { t.Errorf("extractedExecutionName %s is not equal to expected timestamp %s", extractedExecutionName, index_prefix) } } + +func TestExtractAccountInformation(t *testing.T) { + const name, id = "Test", "1234567890" + //test for right input + accountInfo := fmt.Sprintf("%s_%s", name, id) + extractedName, extractedId, err := interpolation.ExtractAccountInformation(accountInfo) + if err != nil { + t.Fatalf("error occured while running extractExecutionName e: %s\n", err) + } + + if extractedName != name { + t.Errorf("extractedName %s is not equal to expected name %s", extractedName, name) + } + + if extractedId != id { + t.Errorf("extractedId %s is not equal to expected id %s", extractedId, id) + } + + //test for wrong input + const wrong = "noUnderScore" + _, _, err = interpolation.ExtractAccountInformation(wrong) + if err == nil { + t.Errorf("function returns no error for input without underscore: %s", wrong) + } +} From 1fd641a5efbb858fe0a1b9f5cdef44cae82f3a28 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 9 Jun 2021 15:37:50 +0200 Subject: [PATCH 10/68] #SOS-20 * save arn in resourceID field + add values in test objects --- collector/aws/resources/apigateway.go | 11 ++++++- collector/aws/resources/ec2.go | 11 ++++++- collector/aws/resources/ec2volumes.go | 11 ++++++- collector/aws/resources/elasticache.go | 11 ++++++- collector/aws/resources/elasticips.go | 34 ++++++++++++++-------- collector/aws/resources/elasticips_test.go | 7 +++-- collector/aws/resources/elb.go | 11 ++++++- collector/aws/resources/elbv2.go | 2 +- collector/aws/resources/iam.go | 2 ++ collector/aws/resources/iam_test.go | 14 +++++++-- collector/aws/resources/kinesis.go | 2 +- collector/aws/resources/natgateway.go | 11 ++++++- collector/aws/resources/redshift.go | 11 ++++++- 13 files changed, 112 insertions(+), 26 deletions(-) diff --git a/collector/aws/resources/apigateway.go b/collector/aws/resources/apigateway.go index 118e5c65..ea79eaec 100644 --- a/collector/aws/resources/apigateway.go +++ b/collector/aws/resources/apigateway.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -144,10 +145,18 @@ func (ag *APIGatewayManager) Detect(metrics []config.MetricConfig) (interface{}, } } + Arn := "arn:aws:apigateway:" + ag.awsManager.GetRegion() + "::/restapis/" + *api.Id + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + detect := DetectedAPIGateway{ Region: ag.awsManager.GetRegion(), Metric: metric.Description, - ResourceID: *api.Id, + ResourceID: Arn, Name: *api.Name, LaunchTime: *api.CreatedDate, Tag: tagsData, diff --git a/collector/aws/resources/ec2.go b/collector/aws/resources/ec2.go index 52fe9f70..e4ba0d13 100644 --- a/collector/aws/resources/ec2.go +++ b/collector/aws/resources/ec2.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "github.com/aws/aws-sdk-go/aws/arn" "strings" "time" @@ -156,13 +157,21 @@ func (ec *EC2Manager) Detect(metrics []config.MetricConfig) (interface{}, error) } } + Arn := "arn:aws:ec2:" + ec.awsManager.GetRegion() + ":" + *ec.awsManager.GetAccountIdentity().Account + ":instance/" + *instance.InstanceId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + ec2 := DetectedEC2{ Region: ec.awsManager.GetRegion(), Metric: metric.Description, Name: name, InstanceType: *instance.InstanceType, PriceDetectedFields: collector.PriceDetectedFields{ - ResourceID: *instance.InstanceId, + ResourceID: Arn, LaunchTime: *instance.LaunchTime, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, diff --git a/collector/aws/resources/ec2volumes.go b/collector/aws/resources/ec2volumes.go index b4a5ccc0..cdf1908a 100644 --- a/collector/aws/resources/ec2volumes.go +++ b/collector/aws/resources/ec2volumes.go @@ -6,6 +6,7 @@ import ( "finala/collector/aws/common" "finala/collector/aws/register" "finala/collector/config" + "github.com/aws/aws-sdk-go/aws/arn" awsClient "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -119,11 +120,19 @@ func (ev *EC2VolumeManager) Detect(metrics []config.MetricConfig) (interface{}, } } + Arn := "arn:aws:ec2:" + ev.awsManager.GetRegion() + ":" + *ev.awsManager.GetAccountIdentity().Account + ":volume/" + *vol.VolumeId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + volumeSize := *vol.Size dEBS := DetectedAWSEC2Volume{ Region: ev.awsManager.GetRegion(), Metric: metric.Description, - ResourceID: *vol.VolumeId, + ResourceID: Arn, Type: *vol.VolumeType, Size: volumeSize, PricePerMonth: ev.getCalculatedPrice(vol, price), diff --git a/collector/aws/resources/elasticache.go b/collector/aws/resources/elasticache.go index 96c9c4c7..06f42fe8 100644 --- a/collector/aws/resources/elasticache.go +++ b/collector/aws/resources/elasticache.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -154,6 +155,14 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} } } + Arn := "arn:aws:elasticache:" + ec.awsManager.GetRegion() + ":" + *ec.awsManager.GetAccountIdentity().Account + ":cluster:" + *instance.CacheClusterId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + es := DetectedElasticache{ Region: ec.awsManager.GetRegion(), Metric: metric.Description, @@ -162,7 +171,7 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} CacheNodes: len(instance.CacheNodes), PriceDetectedFields: collector.PriceDetectedFields{ LaunchTime: *instance.CacheClusterCreateTime, - ResourceID: *instance.CacheClusterId, + ResourceID: Arn, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, diff --git a/collector/aws/resources/elasticips.go b/collector/aws/resources/elasticips.go index 27336e3c..dbc7d8a0 100644 --- a/collector/aws/resources/elasticips.go +++ b/collector/aws/resources/elasticips.go @@ -6,6 +6,7 @@ import ( "finala/collector/aws/common" "finala/collector/aws/register" "finala/collector/config" + "github.com/aws/aws-sdk-go/aws/arn" awsClient "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -29,12 +30,10 @@ type ElasticIPManager struct { // DetectedElasticIP defines the detected AWS elastic ip type DetectedElasticIP struct { - Region string - Metric string - IP string - PricePerHour float64 - PricePerMonth float64 - Tag map[string]string + Region string + Metric string + IP string + collector.PriceDetectedFields collector.AccountSpecifiedFields } @@ -112,13 +111,24 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, } } + Arn := "arn:aws:ec2:" + ei.awsManager.GetRegion() + ":" + *ei.awsManager.GetAccountIdentity().Account + ":elastic-ip/" + *ip.AllocationId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + eIP := DetectedElasticIP{ - Region: ei.awsManager.GetRegion(), - Metric: metric.Description, - IP: *ip.PublicIp, - PricePerHour: price, - PricePerMonth: price * collector.TotalMonthHours, - Tag: tagsData, + Region: ei.awsManager.GetRegion(), + Metric: metric.Description, + IP: *ip.PublicIp, + PriceDetectedFields: collector.PriceDetectedFields{ + ResourceID: Arn, + PricePerHour: price, + PricePerMonth: price * collector.TotalMonthHours, + Tag: tagsData, + }, AccountSpecifiedFields: collector.AccountSpecifiedFields{ AccountID: *ei.awsManager.GetAccountIdentity().Account, AccountName: ei.awsManager.GetAccountName(), diff --git a/collector/aws/resources/elasticips_test.go b/collector/aws/resources/elasticips_test.go index cce26355..56e14de5 100644 --- a/collector/aws/resources/elasticips_test.go +++ b/collector/aws/resources/elasticips_test.go @@ -21,12 +21,15 @@ var defaultAddressesMock = ec2.DescribeAddressesOutput{ AssociationId: awsClient.String("foo-00000"), InstanceId: awsClient.String("i-00000"), NetworkInterfaceId: awsClient.String("00000"), + AllocationId: awsClient.String("aws_eip.example.id"), }, { - PublicIp: awsClient.String("80.80.80.81"), + PublicIp: awsClient.String("80.80.80.81"), + AllocationId: awsClient.String("aws_eip.example.id"), }, { - PublicIp: awsClient.String("80.80.80.82"), + PublicIp: awsClient.String("80.80.80.82"), + AllocationId: awsClient.String("aws_eip.example.id"), }, }, } diff --git a/collector/aws/resources/elb.go b/collector/aws/resources/elb.go index 14b77d1c..e64c20c8 100644 --- a/collector/aws/resources/elb.go +++ b/collector/aws/resources/elb.go @@ -8,6 +8,7 @@ import ( "finala/collector/config" "finala/expression" "fmt" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -170,11 +171,19 @@ func (el *ELBManager) Detect(metrics []config.MetricConfig) (interface{}, error) } } + Arn := "arn:aws:elasticloadbalancing:" + el.awsManager.GetRegion() + ":" + *el.awsManager.GetAccountIdentity().Account + ":loadbalancer/" + *instance.LoadBalancerName + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + elb := DetectedELB{ Region: el.awsManager.GetRegion(), Metric: metric.Description, PriceDetectedFields: collector.PriceDetectedFields{ - ResourceID: *instance.LoadBalancerName, + ResourceID: Arn, LaunchTime: *instance.CreatedTime, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, diff --git a/collector/aws/resources/elbv2.go b/collector/aws/resources/elbv2.go index 4bc5e2b4..ec6eac89 100644 --- a/collector/aws/resources/elbv2.go +++ b/collector/aws/resources/elbv2.go @@ -217,7 +217,7 @@ func (el *ELBV2Manager) Detect(metrics []config.MetricConfig) (interface{}, erro Metric: metric.Description, Type: *instance.Type, PriceDetectedFields: collector.PriceDetectedFields{ - ResourceID: *instance.LoadBalancerName, + ResourceID: *instance.LoadBalancerArn, LaunchTime: *instance.CreatedTime, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, diff --git a/collector/aws/resources/iam.go b/collector/aws/resources/iam.go index dcd12e66..ad5ac99d 100644 --- a/collector/aws/resources/iam.go +++ b/collector/aws/resources/iam.go @@ -34,6 +34,7 @@ type DetectedAWSLastActivity struct { AccessKey string LastUsedDate time.Time LastActivity string + ResourceID string collector.AccountSpecifiedFields } @@ -137,6 +138,7 @@ func (im *IAMManager) Detect(metrics []config.MetricConfig) (interface{}, error) AccessKey: *accessKeyData.AccessKeyId, LastUsedDate: lastUsedDate, LastActivity: lastActivity, + ResourceID: *user.Arn, AccountSpecifiedFields: collector.AccountSpecifiedFields{ AccountID: *im.awsManager.GetAccountIdentity().Account, AccountName: im.awsManager.GetAccountName(), diff --git a/collector/aws/resources/iam_test.go b/collector/aws/resources/iam_test.go index cb6a6813..c1616e6b 100644 --- a/collector/aws/resources/iam_test.go +++ b/collector/aws/resources/iam_test.go @@ -16,9 +16,17 @@ import ( var defaultUsersMock = iam.ListUsersOutput{ Users: []*iam.User{ - {UserName: awsClient.String("foo")}, - {UserName: awsClient.String("foo2")}, - {UserName: awsClient.String("test")}, + { + UserName: awsClient.String("foo"), + Arn: awsClient.String("arn:aws:iam::123456789012:user/foo")}, + { + UserName: awsClient.String("foo2"), + Arn: awsClient.String("arn:aws:iam::123456789012:user/foo2"), + }, + { + UserName: awsClient.String("test"), + Arn: awsClient.String("arn:aws:iam::123456789012:user/test"), + }, }, } diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index b5471346..16c6e571 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -191,7 +191,7 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er Region: km.awsManager.GetRegion(), Metric: metric.Description, PriceDetectedFields: collector.PriceDetectedFields{ - ResourceID: *stream.StreamName, + ResourceID: *stream.StreamARN, LaunchTime: *stream.StreamCreationTimestamp, PricePerHour: totalShardsPerHourPrice, PricePerMonth: totalShardsPerHourPrice * collector.TotalMonthHours, diff --git a/collector/aws/resources/natgateway.go b/collector/aws/resources/natgateway.go index ded329f9..52ceac21 100644 --- a/collector/aws/resources/natgateway.go +++ b/collector/aws/resources/natgateway.go @@ -8,6 +8,7 @@ import ( "finala/collector/config" "finala/expression" "fmt" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -170,6 +171,14 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} } } + Arn := "arn:aws:ec2:" + ngw.awsManager.GetRegion() + ":" + *ngw.awsManager.GetAccountIdentity().Account + ":natgateway/" + *natgateway.NatGatewayId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + natGateway := DetectedNATGateway{ Region: ngw.awsManager.GetRegion(), Metric: metric.Description, @@ -177,7 +186,7 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} VPCID: *natgateway.VpcId, PriceDetectedFields: collector.PriceDetectedFields{ LaunchTime: *natgateway.CreateTime, - ResourceID: *natgateway.NatGatewayId, + ResourceID: Arn, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, diff --git a/collector/aws/resources/redshift.go b/collector/aws/resources/redshift.go index 4dc3d7e0..e791fac3 100644 --- a/collector/aws/resources/redshift.go +++ b/collector/aws/resources/redshift.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -151,6 +152,14 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, } } + Arn := "arn:aws:redshift:" + rdm.awsManager.GetRegion() + ":" + *rdm.awsManager.GetAccountIdentity().Account + ":cluster:" + *cluster.ClusterIdentifier + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + redshift := DetectedRedShift{ Region: rdm.awsManager.GetRegion(), Metric: metric.Description, @@ -158,7 +167,7 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, NumberOfNodes: *cluster.NumberOfNodes, PriceDetectedFields: collector.PriceDetectedFields{ LaunchTime: *cluster.ClusterCreateTime, - ResourceID: *cluster.ClusterIdentifier, + ResourceID: Arn, PricePerHour: clusterPrice, PricePerMonth: clusterPrice * collector.TotalMonthHours, Tag: tagsData, From 9c157f214b6343853f2182a027415269082f5eb4 Mon Sep 17 00:00:00 2001 From: daniebrill Date: Wed, 9 Jun 2021 19:46:46 +0200 Subject: [PATCH 11/68] #SOS-23 Add test for elasticsearch GetAccounts --- api/server_test.go | 2 +- .../elasticsearch/elasticsearch_test.go | 58 +++++++++++++++++++ .../accounts/aggregations/default.json | 12 ++++ interpolation/interpolation_test.go | 4 +- 4 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 api/storage/elasticsearch/testutils/responses/accounts/aggregations/default.json diff --git a/api/server_test.go b/api/server_test.go index a7cad068..55192ec4 100644 --- a/api/server_test.go +++ b/api/server_test.go @@ -279,7 +279,7 @@ func TestGetAccounts(t *testing.T) { } if len(accountsData) != test.Count { - t.Fatalf("unexpected resources summary response, got %d expected %d", len(accountsData), test.Count) + t.Fatalf("unexpected accounts data response, got %d expected %d", len(accountsData), test.Count) } } }) diff --git a/api/storage/elasticsearch/elasticsearch_test.go b/api/storage/elasticsearch/elasticsearch_test.go index d3010bb3..5f78e112 100644 --- a/api/storage/elasticsearch/elasticsearch_test.go +++ b/api/storage/elasticsearch/elasticsearch_test.go @@ -217,6 +217,64 @@ func TestGetExecutions(t *testing.T) { } } +func TestGetAccounts(t *testing.T) { + + // each different queryLimit will result in a different elasticsearch response. + // 1 - returns valid account data response + // 2 - returns invalid aggregation term query response + // 4 - returns invalid statuscode response + testCases := []struct { + name string + queryLimit int + responseCount int + ErrorMessage error + }{ + {"valid response", 1, 2, nil}, + {"invalid terms", 2, 0, ErrAggregationTermNotFound}, + {"invalid es response", 3, 0, ErrInvalidQuery}, + } + + mockClient, config := testutils.NewESMock(prefixIndexName, true) + + mockClient.Router.HandleFunc("/_search", func(resp http.ResponseWriter, req *http.Request) { + switch testutils.GetPostParams(req) { + case `{"aggregations":{"Accounts":{"terms":{"field":"Data.AccountInformation.keyword","size":1}}},"query":{"match":{"ExecutionID":{"query":"1"}}}}`: + testutils.JSONResponse(resp, http.StatusOK, elastic.SearchResult{Aggregations: map[string]json.RawMessage{ + "Accounts": testutils.LoadResponse("accounts/aggregations/default"), + }}) + case `{"aggregations":{"Accounts":{"terms":{"field":"Data.AccountInformation.keyword","size":2}}},"query":{"match":{"ExecutionID":{"query":"1"}}}}`: + testutils.JSONResponse(resp, http.StatusOK, elastic.SearchResult{Aggregations: map[string]json.RawMessage{ + "invalid-key": testutils.LoadResponse("accounts/aggregations/default"), + }}) + case `{"aggregations":{"Accounts":{"terms":{"field":"Data.AccountInformation.keyword","size":3}}},"query":{"match":{"ExecutionID":{"query":"1"}}}}`: + testutils.JSONResponse(resp, http.StatusBadRequest, elastic.SearchResult{Aggregations: map[string]json.RawMessage{}}) + default: + t.Fatalf("unexpected request params") + } + }) + + es, err := NewStorageManager(config) + if err != nil { + t.Fatalf("unexpected error, got %v expected nil", err) + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + + response, err := es.GetAccounts("1", test.queryLimit) + + if err != test.ErrorMessage { + t.Fatalf("unexpected error, got %v expected %v", err, test.ErrorMessage) + } + + if len(response) != test.responseCount { + t.Fatalf("handler query response: got %d want %d", len(response), test.responseCount) + } + + }) + } +} + func TestGetSummary(t *testing.T) { mockClient, config := testutils.NewESMock(prefixIndexName, true) diff --git a/api/storage/elasticsearch/testutils/responses/accounts/aggregations/default.json b/api/storage/elasticsearch/testutils/responses/accounts/aggregations/default.json new file mode 100644 index 00000000..ff15d0c8 --- /dev/null +++ b/api/storage/elasticsearch/testutils/responses/accounts/aggregations/default.json @@ -0,0 +1,12 @@ +{ + "buckets" : [ + { + "key" : "Test_123456789", + "doc_count" : 66 + }, + { + "key" : "Test2_12345675", + "doc_count" : 6 + } + ] +} \ No newline at end of file diff --git a/interpolation/interpolation_test.go b/interpolation/interpolation_test.go index 47612441..221c64fd 100644 --- a/interpolation/interpolation_test.go +++ b/interpolation/interpolation_test.go @@ -78,7 +78,7 @@ func TestExtractAccountInformation(t *testing.T) { accountInfo := fmt.Sprintf("%s_%s", name, id) extractedName, extractedId, err := interpolation.ExtractAccountInformation(accountInfo) if err != nil { - t.Fatalf("error occured while running extractExecutionName e: %s\n", err) + t.Fatalf("error occured while running ExtractAccountInformation e: %s\n", err) } if extractedName != name { @@ -93,6 +93,6 @@ func TestExtractAccountInformation(t *testing.T) { const wrong = "noUnderScore" _, _, err = interpolation.ExtractAccountInformation(wrong) if err == nil { - t.Errorf("function returns no error for input without underscore: %s", wrong) + t.Errorf("ExtractAccountInformation returns no error for input without underscore: %s", wrong) } } From 0f68ad2b2b5a5f3ff952d6e24b71eea42d98a06f Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Thu, 10 Jun 2021 15:57:59 +0200 Subject: [PATCH 12/68] #SOS-16 + add accounts to the provider store + Show Accounts in a chip-list like resources + add Accounts to filter --- ui/src/components/Dashboard/AccountsList.js | 82 +++++++++++++++++++++ ui/src/components/Dashboard/Index.js | 2 + ui/src/components/DataFactory.js | 21 ++++++ ui/src/reducers/accounts.reducer.js | 18 +++++ ui/src/reducers/index.js | 2 + ui/src/services/accs.service.js | 18 +++++ 6 files changed, 143 insertions(+) create mode 100644 ui/src/components/Dashboard/AccountsList.js create mode 100644 ui/src/reducers/accounts.reducer.js create mode 100644 ui/src/services/accs.service.js diff --git a/ui/src/components/Dashboard/AccountsList.js b/ui/src/components/Dashboard/AccountsList.js new file mode 100644 index 00000000..10bfa337 --- /dev/null +++ b/ui/src/components/Dashboard/AccountsList.js @@ -0,0 +1,82 @@ +import React, { Fragment } from "react"; +import { connect } from "react-redux"; +import PropTypes from "prop-types"; +import colors from "./colors.json"; +import { makeStyles } from "@material-ui/core/styles"; +import { Box, Chip } from "@material-ui/core"; +import { setHistory } from "../../utils/History"; + +const useStyles = makeStyles(() => ({ + title: { + fontFamily: "MuseoModerno", + }, + resource_chips: { + fontWeight: "bold", + fontFamily: "Arial !important", + margin: "5px", + borderRadius: "1px", + backgroundColor: "#ffffff", + borderLeft: "5px solid #ffffff", + fontSize: "14px", + }, +})); + +const AccountsList = ({ accounts, filters, addFilter }) => { + const classes = useStyles(); + + const accountsList = Object.values(accounts).map((account) => { + account.title = `${account.Name}(${account.ID})`; + return account; + }); + + const setSelectedAccount = (account) => { + const filter = { + title: `Account:${account.title}`, + id: `account:${account.ID}`, + type: "account", + }; + + addFilter(filter); + + setHistory({ + filters: filters, + }); + }; + + return ( + + {accountsList.length > 0 && ( + +

Accounts:

+ {accountsList.map((account, i) => ( + setSelectedAccount(account)} + ma={2} + label={account.title} + key={i} + /> + ))} + + )} + + ); +}; + +AccountsList.defaultProps = {}; +AccountsList.propTypes = { + accounts: PropTypes.object, + filters: PropTypes.array, + addFilter: PropTypes.func, +}; + +const mapStateToProps = (state) => ({ + accounts: state.accounts.accounts, + filters: state.filters.filters, +}); +const mapDispatchToProps = (dispatch) => ({ + addFilter: (data) => dispatch({ type: "ADD_FILTER", data }), +}); + +export default connect(mapStateToProps, mapDispatchToProps)(AccountsList); diff --git a/ui/src/components/Dashboard/Index.js b/ui/src/components/Dashboard/Index.js index a137f8e9..eb8a80f7 100644 --- a/ui/src/components/Dashboard/Index.js +++ b/ui/src/components/Dashboard/Index.js @@ -4,6 +4,7 @@ import { makeStyles } from "@material-ui/core/styles"; import { setHistory } from "../../utils/History"; import PropTypes from "prop-types"; +import AccountsList from "./AccountsList"; import FilterBar from "./FilterBar"; import StatisticsBar from "./StatisticsBar"; import ResourceScanning from "./ResourceScanning"; @@ -71,6 +72,7 @@ const DashboardIndex = ({ + {currentResource ? : } diff --git a/ui/src/components/DataFactory.js b/ui/src/components/DataFactory.js index dd050fd1..ab440a11 100644 --- a/ui/src/components/DataFactory.js +++ b/ui/src/components/DataFactory.js @@ -5,6 +5,7 @@ import { ResourcesService } from "services/resources.service"; import { SettingsService } from "services/settings.service"; import { titleDirective } from "utils/Title"; import { getHistory, setHistory } from "../utils/History"; +import { AccsService } from "../services/accs.service"; let fetchTimeoutRequest = false; let fetchTableTimeoutRequest = false; @@ -38,6 +39,7 @@ const DataFacotry = ({ setCurrentExecution, currentResource, + setAccounts, setResources, setCurrentResourceData, setIsResourceListLoading, @@ -114,6 +116,7 @@ const DataFacotry = ({ clearTimeout(fetchTimeoutRequest); setIsResourceListLoading(true); await getResources(currentExecution, filters); + await getAccounts(currentExecution); setIsResourceListLoading(false); if (currentResource) { @@ -140,6 +143,20 @@ const DataFacotry = ({ setIsResourceTableLoading(false); }; + const getAccounts = async (currentExecution) => { + const AccountsArray = await AccsService.list(currentExecution).catch( + () => false + ); + + const accounts = {}; + AccountsArray.forEach((value) => { + accounts[value.ID] = value; + }); + + setAccounts(accounts); + return true; + }; + /** * Will fetch resource list from server * @param {string} currentExecution Current Selected Execution @@ -258,11 +275,13 @@ DataFacotry.propTypes = { setIsResourceListLoading: PropTypes.func, setIsResourceTableLoading: PropTypes.func, setIsScanning: PropTypes.func, + setAccounts: PropTypes.func, setResources: PropTypes.func, setCurrentResourceData: PropTypes.func, setCurrentExecution: PropTypes.func, currentResource: PropTypes.string, + accounts: PropTypes.object, resources: PropTypes.object, filters: PropTypes.array, currentExecution: PropTypes.string, @@ -273,6 +292,7 @@ DataFacotry.propTypes = { }; const mapStateToProps = (state) => ({ + accounts: state.accounts.accounts, resources: state.resources.resources, currentResource: state.resources.currentResource, currentExecution: state.executions.current, @@ -289,6 +309,7 @@ const mapDispatchToProps = (dispatch) => ({ setIsResourceTableLoading: (isLoading) => dispatch({ type: "IS_RESOURCE_TABLE_LOADING", isLoading }), setIsScanning: (isScanning) => dispatch({ type: "IS_SCANNING", isScanning }), + setAccounts: (data) => dispatch({ type: "ACCOUNT_LIST", data }), setResources: (data) => dispatch({ type: "RESOURCE_LIST", data }), setCurrentExecution: (id) => dispatch({ type: "EXECUTION_SELECTED", id }), setCurrentResourceData: (data) => diff --git a/ui/src/reducers/accounts.reducer.js b/ui/src/reducers/accounts.reducer.js new file mode 100644 index 00000000..b88b24ff --- /dev/null +++ b/ui/src/reducers/accounts.reducer.js @@ -0,0 +1,18 @@ +const initialState = { + accounts: {}, +}; + +/** + * @param {object} state module state + * @param {object} action to apply on state + * @returns {object} new copy of state + */ +export function accounts(state = initialState, action) { + switch (action.type) { + case "ACCOUNT_LIST": + state.accounts = action.data; + return { ...state }; + default: + return state; + } +} diff --git a/ui/src/reducers/index.js b/ui/src/reducers/index.js index f894e2dd..ad94d29c 100755 --- a/ui/src/reducers/index.js +++ b/ui/src/reducers/index.js @@ -1,11 +1,13 @@ import { combineReducers } from "redux"; import { connectRouter } from "connected-react-router"; +import { accounts } from "../reducers/accounts.reducer"; import { resources } from "../reducers/resources.reducer"; import { executions } from "../reducers/executions.reducer"; import { filters } from "../reducers/filters.reducer"; const rootReducer = (history) => combineReducers({ + accounts, resources, executions, filters, diff --git a/ui/src/services/accs.service.js b/ui/src/services/accs.service.js new file mode 100644 index 00000000..89666cd3 --- /dev/null +++ b/ui/src/services/accs.service.js @@ -0,0 +1,18 @@ +import { http } from "./request.service"; + +export const AccsService = { + list, +}; + +/** + * + * @param {string} executionId execution to query + */ +function list(executionId) { + return http + .send(`api/v1/accounts/${executionId}`, `get`) + .then(this.handleResponse) + .then((response) => { + return response; + }); +} From bf4c57d29327b0d56f30d54ca32935a3a9f455ba Mon Sep 17 00:00:00 2001 From: daniebrill Date: Fri, 11 Jun 2021 15:50:45 +0200 Subject: [PATCH 13/68] #SOS18 Add multiaccount selection functionality to tables --- ui/src/components/Dashboard/FilterBar.js | 7 +++++++ ui/src/services/resources.service.js | 9 +++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ui/src/components/Dashboard/FilterBar.js b/ui/src/components/Dashboard/FilterBar.js index ca2f9137..f9822f20 100644 --- a/ui/src/components/Dashboard/FilterBar.js +++ b/ui/src/components/Dashboard/FilterBar.js @@ -137,6 +137,13 @@ const FilterBar = ({ type: "resource", }); resource = filterValue; + } else if (filterValue && filterKey === "account") { + filters.push({ + title: `Account:${filterValue}`, + id: filter, + value: filterValue, + type: "account", + }); } else if (filterValue) { const filterValues = filterValue.split(","); diff --git a/ui/src/services/resources.service.js b/ui/src/services/resources.service.js index 878aa72b..8ab79481 100644 --- a/ui/src/services/resources.service.js +++ b/ui/src/services/resources.service.js @@ -14,12 +14,17 @@ export const ResourcesService = { const getTransformedFilters = (filters) => { const params = {}; filters.forEach((filter) => { - if (filter.id.substr(0, 8) === "resource") { + if (filter.type === "resource") { return; } const [key, value] = filter.id.split(":"); + let paramKey; + if (value && filter.type === "account") { + paramKey = `filter_Data.AccountID`; + } else { + paramKey = `filter_Data.Tag.${key}`; + } if (value) { - const paramKey = `filter_Data.Tag.${key}`; if (params[paramKey]) { params[paramKey] += `,${value}`; } else { From ecbe7ef1e81cbe7736079565565bd7409f5056da Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Fri, 11 Jun 2021 15:57:50 +0200 Subject: [PATCH 14/68] #SOS-16 add comments --- ui/src/components/Dashboard/AccountsList.js | 9 +++++++++ ui/src/components/DataFactory.js | 5 +++++ ui/src/services/resources.service.js | 1 + 3 files changed, 15 insertions(+) diff --git a/ui/src/components/Dashboard/AccountsList.js b/ui/src/components/Dashboard/AccountsList.js index 10bfa337..025b0170 100644 --- a/ui/src/components/Dashboard/AccountsList.js +++ b/ui/src/components/Dashboard/AccountsList.js @@ -21,6 +21,11 @@ const useStyles = makeStyles(() => ({ }, })); +/** + * @param {array} accounts Accounts List + * @param {array} filters Filters List + * @param {func} addFilter Add filter to filters list + */ const AccountsList = ({ accounts, filters, addFilter }) => { const classes = useStyles(); @@ -29,6 +34,10 @@ const AccountsList = ({ accounts, filters, addFilter }) => { return account; }); + /** + * + * @param {object} account add selected account + */ const setSelectedAccount = (account) => { const filter = { title: `Account:${account.title}`, diff --git a/ui/src/components/DataFactory.js b/ui/src/components/DataFactory.js index ab440a11..1e90ccf0 100644 --- a/ui/src/components/DataFactory.js +++ b/ui/src/components/DataFactory.js @@ -20,6 +20,7 @@ let lastFiltersSearched = "[]"; * @param {func} setCurrentExecution Update Current Execution * * @param {string} currentResource Current selected resource + * @param {func} setAccounts Update Accounts List * @param {func} setResources Update Resources List * @param {func} setCurrentResourceData Update current resource data * @param {func} setIsResourceListLoading update isLoading state for resources @@ -143,6 +144,10 @@ const DataFacotry = ({ setIsResourceTableLoading(false); }; + /** + * Will fetch account list from server + * @param {string} currentExecution current Selected Execution + */ const getAccounts = async (currentExecution) => { const AccountsArray = await AccsService.list(currentExecution).catch( () => false diff --git a/ui/src/services/resources.service.js b/ui/src/services/resources.service.js index 878aa72b..c807f439 100644 --- a/ui/src/services/resources.service.js +++ b/ui/src/services/resources.service.js @@ -12,6 +12,7 @@ export const ResourcesService = { * @returns filters params for request */ const getTransformedFilters = (filters) => { + console.log(filters); const params = {}; filters.forEach((filter) => { if (filter.id.substr(0, 8) === "resource") { From 5ea12e5b721cfaf648c2d08dfe3093b5dd10cd8d Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Mon, 14 Jun 2021 16:47:54 +0200 Subject: [PATCH 15/68] #SOS-18 add spentAccounts in summary add things to success tests --- api/storage/elasticsearch/elasticsearch.go | 47 +++++++++++-------- .../elasticsearch/elasticsearch_test.go | 7 ++- .../summary/aggregations/default.json | 11 +++++ api/storage/structs.go | 13 ++--- 4 files changed, 52 insertions(+), 26 deletions(-) create mode 100644 api/storage/elasticsearch/testutils/responses/summary/aggregations/default.json diff --git a/api/storage/elasticsearch/elasticsearch.go b/api/storage/elasticsearch/elasticsearch.go index 0fd4403b..471d1d55 100644 --- a/api/storage/elasticsearch/elasticsearch.go +++ b/api/storage/elasticsearch/elasticsearch.go @@ -194,7 +194,7 @@ func (sm *StorageManager) GetSummary(executionID string, filters map[string]stri for resourceName, resourceData := range summary { filters["ResourceName"] = resourceName log.WithField("filters", filters).Debug("Going to get resources summary details with the following filters") - totalSpent, resourceCount, err := sm.getResourceSummaryDetails(executionID, filters) + totalSpent, resourceCount, spentAccounts, err := sm.getResourceSummaryDetails(executionID, filters) if err != nil { continue @@ -202,8 +202,8 @@ func (sm *StorageManager) GetSummary(executionID string, filters map[string]stri newResourceData := resourceData newResourceData.TotalSpent = totalSpent newResourceData.ResourceCount = resourceCount + newResourceData.SpentAccounts = spentAccounts summary[resourceName] = newResourceData - } return summary, nil @@ -211,43 +211,52 @@ func (sm *StorageManager) GetSummary(executionID string, filters map[string]stri } // getResourceSummaryDetails returns total resource spent and total resources detected -func (sm *StorageManager) getResourceSummaryDetails(executionID string, filters map[string]string) (float64, int64, error) { +func (sm *StorageManager) getResourceSummaryDetails(executionID string, filters map[string]string) (float64, int64, map[string]float64, error) { var totalSpent float64 var resourceCount int64 + var spentAccounts = make(map[string]float64) dynamicMatchQuery := sm.getDynamicMatchQuery(filters, "or") dynamicMatchQuery = append(dynamicMatchQuery, elastic.NewTermQuery("ExecutionID", executionID)) dynamicMatchQuery = append(dynamicMatchQuery, elastic.NewTermQuery("EventType", "resource_detected")) - searchResult, err := sm.client.Search(). + searchResultAccount, err := sm.client.Search(). Query(elastic.NewBoolQuery().Must(dynamicMatchQuery...)). - Aggregation("sum", elastic.NewSumAggregation().Field("Data.PricePerMonth")). + Aggregation("accounts", elastic.NewTermsAggregation().Field("Data.AccountID.keyword"). + SubAggregation("accountSum", elastic.NewSumAggregation().Field("Data.PricePerMonth"))). Size(0).Do(context.Background()) if err != nil { - log.WithError(err).WithFields(log.Fields{ - "filters": filters, - }).Error("error when trying to get summary details") + log.WithError(err).Error("error when trying to get executions collectors") + return totalSpent, resourceCount, spentAccounts, ErrInvalidQuery + } - return totalSpent, resourceCount, err + respAccount, ok := searchResultAccount.Aggregations.Terms("accounts") + if !ok { + log.Error("accounts field term does not exist") + return totalSpent, resourceCount, spentAccounts, ErrAggregationTermNotFound } - log.WithFields(log.Fields{ - "filters": filters, - "milliseconds": searchResult.TookInMillis, - }).Debug("get execution details") + for _, AccountIdBucket := range respAccount.Buckets { - resp, ok := searchResult.Aggregations.Terms("sum") - if ok { - if val, ok := resp.Aggregations["value"]; ok { + spent, ok := AccountIdBucket.Aggregations.Terms("accountSum") + if ok { + if val, ok := spent.Aggregations["value"]; ok { + accountID, ok := AccountIdBucket.Key.(string) + if !ok { + log.Error("type assertion to string failed") + continue + } + spentAccounts[accountID], _ = strconv.ParseFloat(string(val), 64) - totalSpent, _ = strconv.ParseFloat(string(val), 64) - resourceCount = searchResult.Hits.TotalHits.Value + totalSpent += spentAccounts[accountID] + resourceCount += AccountIdBucket.DocCount + } } } - return totalSpent, resourceCount, nil + return totalSpent, resourceCount, spentAccounts, nil } // GetExecutions returns collector executions diff --git a/api/storage/elasticsearch/elasticsearch_test.go b/api/storage/elasticsearch/elasticsearch_test.go index 5f78e112..598e2e7d 100644 --- a/api/storage/elasticsearch/elasticsearch_test.go +++ b/api/storage/elasticsearch/elasticsearch_test.go @@ -283,7 +283,9 @@ func TestGetSummary(t *testing.T) { response := elastic.SearchResult{} - switch testutils.GetPostParams(req) { + var s = testutils.GetPostParams(req) + fmt.Println(s) + switch s { case `{"query":{"bool":{"must":[{"term":{"EventType":"service_status"}},{"term":{"ExecutionID":""}}]}},"size":0}`: response.Hits = &elastic.SearchHits{TotalHits: &elastic.TotalHits{Value: 1}} case `{"query":{"bool":{"must":[{"term":{"EventType":"service_status"}},{"term":{"ExecutionID":""}}]}},"size":1}`: @@ -296,6 +298,9 @@ func TestGetSummary(t *testing.T) { case `{"aggregations":{"sum":{"sum":{"field":"Data.PricePerMonth"}}},"query":{"bool":{"must":[{"match":{"ResourceName":{"minimum_should_match":"100%","query":"aws_resource_name"}}},{"term":{"ExecutionID":""}},{"term":{"EventType":"resource_detected"}}]}},"size":0}`: response.Aggregations = map[string]json.RawMessage{"sum": []byte(`{"value": 36.5}`)} response.Hits = &elastic.SearchHits{TotalHits: &elastic.TotalHits{Value: 1}} + case `{"aggregations":{"accounts":{"aggregations":{"accountSum":{"sum":{"field":"Data.PricePerMonth"}}},"terms":{"field":"Data.AccountID.keyword"}}},"query":{"bool":{"must":[{"match":{"ResourceName":{"minimum_should_match":"100%","query":"aws_resource_name"}}},{"term":{"ExecutionID":""}},{"term":{"EventType":"resource_detected"}}]}},"size":0}`: + response.Aggregations = map[string]json.RawMessage{"accounts": testutils.LoadResponse("summary/aggregations/default")} + response.Hits = &elastic.SearchHits{TotalHits: &elastic.TotalHits{Value: 1}} default: t.Fatalf("unexpected request params") } diff --git a/api/storage/elasticsearch/testutils/responses/summary/aggregations/default.json b/api/storage/elasticsearch/testutils/responses/summary/aggregations/default.json new file mode 100644 index 00000000..082107a4 --- /dev/null +++ b/api/storage/elasticsearch/testutils/responses/summary/aggregations/default.json @@ -0,0 +1,11 @@ +{ + "buckets": [ + { + "key": "123456789012", + "doc_count": 1, + "accountSum": { + "value": 36.5 + } + } + ] +} \ No newline at end of file diff --git a/api/storage/structs.go b/api/storage/structs.go index a2bef4d6..0d453414 100644 --- a/api/storage/structs.go +++ b/api/storage/structs.go @@ -39,12 +39,13 @@ type Accounts struct { // CollectorsSummary defines unused resource summary type CollectorsSummary struct { - ResourceName string `json:"ResourceName"` - ResourceCount int64 `json:"ResourceCount"` - TotalSpent float64 `json:"TotalSpent"` - Status int `json:"Status"` - ErrorMessage string `json:"ErrorMessage"` - EventTime int64 `json:"-"` + ResourceName string `json:"ResourceName"` + ResourceCount int64 `json:"ResourceCount"` + TotalSpent float64 `json:"TotalSpent"` + Status int `json:"Status"` + ErrorMessage string `json:"ErrorMessage"` + EventTime int64 `json:"-"` + SpentAccounts map[string]float64 `json:"SpentAccounts"` } type SummaryData struct { From 2f995482ba90c3265fb3d1fe5dbb62c1f057db1f Mon Sep 17 00:00:00 2001 From: daniebrill Date: Mon, 14 Jun 2021 18:12:15 +0200 Subject: [PATCH 16/68] #SOS-18 Add multiaccount-selection functionality for RessourceCharts --- ui/src/components/Dashboard/FilterBar.js | 12 ++-- ui/src/components/Dashboard/Index.js | 3 +- ui/src/components/Dashboard/ResourcesChart.js | 56 +++++++++++++++---- .../components/Dashboard/ResourcesCharts.js | 36 ++++++++++++ 4 files changed, 89 insertions(+), 18 deletions(-) create mode 100644 ui/src/components/Dashboard/ResourcesCharts.js diff --git a/ui/src/components/Dashboard/FilterBar.js b/ui/src/components/Dashboard/FilterBar.js index f9822f20..d029cb76 100644 --- a/ui/src/components/Dashboard/FilterBar.js +++ b/ui/src/components/Dashboard/FilterBar.js @@ -138,11 +138,13 @@ const FilterBar = ({ }); resource = filterValue; } else if (filterValue && filterKey === "account") { - filters.push({ - title: `Account:${filterValue}`, - id: filter, - value: filterValue, - type: "account", + const accounts = filterValue.split(","); + accounts.forEach((account) => { + filters.push({ + title: `Account:${account}`, + id: `account:${account}`, + type: "account", + }); }); } else if (filterValue) { const filterValues = filterValue.split(","); diff --git a/ui/src/components/Dashboard/Index.js b/ui/src/components/Dashboard/Index.js index eb8a80f7..cb963357 100644 --- a/ui/src/components/Dashboard/Index.js +++ b/ui/src/components/Dashboard/Index.js @@ -9,6 +9,7 @@ import FilterBar from "./FilterBar"; import StatisticsBar from "./StatisticsBar"; import ResourceScanning from "./ResourceScanning"; import ResourcesChart from "./ResourcesChart"; +import ResourcesCharts from "./ResourcesCharts"; import ResourcesList from "./ResourcesList"; import ResourceTable from "./ResourceTable"; import ExecutionIndex from "../Executions/Index"; @@ -74,7 +75,7 @@ const DashboardIndex = ({ - {currentResource ? : } + {currentResource ? : } ); }; diff --git a/ui/src/components/Dashboard/ResourcesChart.js b/ui/src/components/Dashboard/ResourcesChart.js index 42140213..fa1fa8ce 100644 --- a/ui/src/components/Dashboard/ResourcesChart.js +++ b/ui/src/components/Dashboard/ResourcesChart.js @@ -17,6 +17,9 @@ import { import ReportProblemIcon from "@material-ui/icons/ReportProblem"; const useStyles = makeStyles(() => ({ + title: { + fontFamily: "MuseoModerno", + }, noDataTitle: { textAlign: "center", fontWeight: "bold", @@ -38,6 +41,8 @@ const useStyles = makeStyles(() => ({ * @param {bool} isResourceListLoading isLoading state for resources * @param {func} addFilter Add filter to filters list * @param {func} setResource Update Selected Resource} + * @param {string} account Account ID for account specific summary + * @param {object} accounts Accounts of current execution */ const ResourcesChart = ({ resources, @@ -45,12 +50,25 @@ const ResourcesChart = ({ isResourceListLoading, addFilter, setResource, + account, + accounts, }) => { const classes = useStyles(); const colorList = colors.map((color) => color.hex); - const sortedResources = Object.values(resources) - .filter((row) => row.TotalSpent > 0) - .sort((a, b) => (a.TotalSpent >= b.TotalSpent ? -1 : 1)); + let sortedResources; + if (account) { + sortedResources = Object.values(resources) + .filter( + (row) => row.SpentAccounts[account] && row.SpentAccounts[account] > 0 + ) + .sort((a, b) => + a.SpentAccounts[account] >= b.SpentAccounts[account] ? -1 : 1 + ); + } else { + sortedResources = Object.values(resources) + .filter((row) => row.TotalSpent > 0) + .sort((a, b) => (a.TotalSpent >= b.TotalSpent ? -1 : 1)); + } const chartOptions = { options: { @@ -148,12 +166,16 @@ const ResourcesChart = ({ */ sortedResources.forEach((resource) => { const title = titleDirective(resource.ResourceName); - const amount = MoneyDirective(resource.TotalSpent); + const amount = MoneyDirective( + account ? resource.SpentAccounts[account] : resource.TotalSpent + ); resource.title = `${title} (${amount})`; resource.display_title = `${title}`; chartOptions.options.xaxis.categories.push(resource.title); - chartOptions.series[0].data.push(resource.TotalSpent); + chartOptions.series[0].data.push( + account ? resource.SpentAccounts[account] : resource.TotalSpent + ); return resource; }); @@ -163,13 +185,20 @@ const ResourcesChart = ({ {!isResourceListLoading && sortedResources.length > 0 && ( - + +

+ {account + ? `${accounts[account].Name} (${accounts[account].ID}):` + : "Summary:"} +

+ + )} {isResourceListLoading && ( @@ -194,12 +223,15 @@ ResourcesChart.propTypes = { isResourceListLoading: PropTypes.bool, addFilter: PropTypes.func, setResource: PropTypes.func, + account: PropTypes.string, + accounts: PropTypes.object, }; const mapStateToProps = (state) => ({ resources: state.resources.resources, isResourceListLoading: state.resources.isResourceListLoading, filters: state.filters.filters, + accounts: state.accounts.accounts, }); const mapDispatchToProps = (dispatch) => ({ diff --git a/ui/src/components/Dashboard/ResourcesCharts.js b/ui/src/components/Dashboard/ResourcesCharts.js new file mode 100644 index 00000000..1ba379d6 --- /dev/null +++ b/ui/src/components/Dashboard/ResourcesCharts.js @@ -0,0 +1,36 @@ +import React from "react"; +import { connect } from "react-redux"; +import PropTypes from "prop-types"; +import { Fragment } from "react"; +import ResourcesChart from "./ResourcesChart"; + +/** + * @param {accounts} object Accounts of current execution + * @param {filters} array Filters list + */ +const ResourcesCharts = ({ accounts, filters }) => { + let selectedAccountIds = filters + .filter((filter) => filter.type === "account") + .map((filter) => filter.id.split(":")[1]); + if (selectedAccountIds.length === 0) { + selectedAccountIds = Object.keys(accounts); + } + let resourcesCharts = selectedAccountIds.map((accountID) => ( + + )); + resourcesCharts = [, ...resourcesCharts]; + return {resourcesCharts}; +}; + +ResourcesCharts.defaultProps = {}; +ResourcesCharts.propTypes = { + filters: PropTypes.array, + accounts: PropTypes.object, +}; + +const mapStateToProps = (state) => ({ + filters: state.filters.filters, + accounts: state.accounts.accounts, +}); + +export default connect(mapStateToProps)(ResourcesCharts); From 8416b891700e708468fe9fd0cfe16f91a794e968 Mon Sep 17 00:00:00 2001 From: Florian <26717601+fbthb@users.noreply.github.com> Date: Wed, 16 Jun 2021 16:07:05 +0200 Subject: [PATCH 17/68] #SOS-31 --- configuration/collector.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/configuration/collector.yaml b/configuration/collector.yaml index 95557126..6f8e306c 100644 --- a/configuration/collector.yaml +++ b/configuration/collector.yaml @@ -210,4 +210,15 @@ providers: start_time: 168h # 24h * 7d constraint: operator: "==" - value: 0 + value: 0 + s3: + - description: Number of Requests + enable: true + metrics: + - name: AllRequests + statistic: Sum + period: 24h + start_time: 168h # 24h * 7d + constraint: + operator: "==" + value: 0 From e55138e2644cfcfb51bf53fb201ae6b2881025e4 Mon Sep 17 00:00:00 2001 From: daniebrill Date: Wed, 16 Jun 2021 17:55:31 +0200 Subject: [PATCH 18/68] #SOS-26 Add EKS to collector configuration file --- configuration/collector.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/configuration/collector.yaml b/configuration/collector.yaml index 95557126..4bbd2812 100644 --- a/configuration/collector.yaml +++ b/configuration/collector.yaml @@ -211,3 +211,14 @@ providers: constraint: operator: "==" value: 0 + eks: + - description: Cluster node count + enable: true + metrics: + - name: ClusterNodeCount + statistic: Maximum + period: 24h + start_time: 168h # 24h * 7d + constraint: + operator: "<" + value: 1 \ No newline at end of file From 454eb3aa5e0b068e9a003b53bf9deb0f798a168e Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Thu, 17 Jun 2021 15:59:25 +0200 Subject: [PATCH 19/68] #SOS-25 add EKS Cluster Check with test-file --- collector/aws/resources/eks.go | 235 ++++++++++++++++++++++++++++ collector/aws/resources/eks_test.go | 214 +++++++++++++++++++++++++ 2 files changed, 449 insertions(+) create mode 100644 collector/aws/resources/eks.go create mode 100644 collector/aws/resources/eks_test.go diff --git a/collector/aws/resources/eks.go b/collector/aws/resources/eks.go new file mode 100644 index 00000000..0af874a1 --- /dev/null +++ b/collector/aws/resources/eks.go @@ -0,0 +1,235 @@ +package resources + +import ( + "errors" + "finala/collector" + "finala/collector/aws/common" + "finala/collector/aws/register" + "finala/collector/config" + "finala/expression" + awsClient "github.com/aws/aws-sdk-go/aws" + awsCloudwatch "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/eks" + "github.com/aws/aws-sdk-go/service/pricing" + log "github.com/sirupsen/logrus" + "time" +) + +// EKSClientDescriptor is an interface defining the aws eks client +type EKSClientDescriptor interface { + ListClusters(input *eks.ListClustersInput) (*eks.ListClustersOutput, error) + DescribeCluster(input *eks.DescribeClusterInput) (*eks.DescribeClusterOutput, error) +} + +// EKSManager describes EKS struct +type EKSManager struct { + client EKSClientDescriptor + awsManager common.AWSManager + namespace string + servicePricingCode string + Name collector.ResourceIdentifier +} + +// DetectedEKS define the detected AWS EKS Cluster +type DetectedEKS struct { + Metric string + Region string + collector.PriceDetectedFields + collector.AccountSpecifiedFields +} + +func init() { + register.Registry("eks", NewEKSManager) +} + +// NewEKSManager implements AWS GO SDK +func NewEKSManager(awsManager common.AWSManager, client interface{}) (common.ResourceDetection, error) { + + if client == nil { + client = eks.New(awsManager.GetSession()) + } + + eksClient, ok := client.(EKSClientDescriptor) + if !ok { + return nil, errors.New("invalid eks volumes client") + } + + return &EKSManager{ + client: eksClient, + awsManager: awsManager, + namespace: "AWS/EKS", + servicePricingCode: "AmazonEKS", + Name: awsManager.GetResourceIdentifier("eks"), + }, nil +} + +// Detect check if eks cluster is too empty +func (ek *EKSManager) Detect(metrics []config.MetricConfig) (interface{}, error) { + + log.WithFields(log.Fields{ + "region": ek.awsManager.GetRegion(), + "resource": "eks", + }).Info("analyzing resource") + + ek.awsManager.GetCollector().CollectStart(ek.Name, collector.AccountSpecifiedFields{ + AccountID: *ek.awsManager.GetAccountIdentity().Account, + AccountName: ek.awsManager.GetAccountName(), + }) + + detectedEKSClusters := []DetectedEKS{} + + clusters, err := ek.describeCluster(nil, nil) + if err != nil { + ek.awsManager.GetCollector().CollectError(ek.Name, err) + return detectedEKSClusters, err + } + + now := time.Now() + + for _, cluster := range clusters { + log.WithField("name", *cluster.Name).Debug("checking eks") + + price, _ := ek.awsManager.GetPricingClient().GetPrice(ek.getPricingFilterInput(cluster), "", ek.awsManager.GetRegion()) + + for _, metric := range metrics { + log.WithFields(log.Fields{ + "cluster_name": *cluster.Name, + "metric_name": metric.Description, + }).Debug("check metric") + + period := int64(metric.Period.Seconds()) + metricEndTime := now.Add(time.Duration(-metric.StartTime)) + metricInput := awsCloudwatch.GetMetricStatisticsInput{ + Namespace: &ek.namespace, + MetricName: &metric.Description, + Period: &period, + StartTime: &metricEndTime, + EndTime: &now, + } + + formulaValue, _, err := ek.awsManager.GetCloudWatchClient().GetMetric(&metricInput, metric) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "cluster_name": *cluster.Name, + "metric_name": metric.Description, + }).Error("Could not get cloudwatch metric data") + continue + } + + expression, err := expression.BoolExpression(formulaValue, metric.Constraint.Value, metric.Constraint.Operator) + if err != nil { + log.WithField("error", err).Error("could not parse expression") + continue + } + + if expression { + + log.WithFields(log.Fields{ + "metric_name": metric.Description, + "constraint_operator": metric.Constraint.Operator, + "constraint_Value": metric.Constraint.Value, + "formula_value": formulaValue, + "cluster_name": *cluster.Name, + "region": ek.awsManager.GetRegion(), + }).Info("EKS cluster detected as unutilized resource") + + tagsData := map[string]string{} + if err == nil { + for tagKey, tagValue := range cluster.Tags { + tagsData[tagKey] = *tagValue + } + } + + eks := DetectedEKS{ + Region: ek.awsManager.GetRegion(), + Metric: metric.Description, + PriceDetectedFields: collector.PriceDetectedFields{ + LaunchTime: *cluster.CreatedAt, + ResourceID: *cluster.Arn, + PricePerHour: price, + PricePerMonth: price * collector.TotalMonthHours, + Tag: tagsData, + }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ek.awsManager.GetAccountIdentity().Account, + AccountName: ek.awsManager.GetAccountName(), + }, + } + + ek.awsManager.GetCollector().AddResource(collector.EventCollector{ + ResourceName: collector.ResourceIdentifier(ek.Name), + Data: eks, + }) + + detectedEKSClusters = append(detectedEKSClusters, eks) + } + + } + } + + ek.awsManager.GetCollector().CollectFinish(ek.Name, collector.AccountSpecifiedFields{ + AccountID: *ek.awsManager.GetAccountIdentity().Account, + AccountName: ek.awsManager.GetAccountName(), + }) + + return detectedEKSClusters, nil +} + +func (ek *EKSManager) getPricingFilterInput(cluster *eks.Cluster) pricing.GetProductsInput { + + return pricing.GetProductsInput{ + ServiceCode: &ek.servicePricingCode, + Filters: []*pricing.Filter{ + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("clusterName"), + Value: cluster.Name, + }, + }, + } + +} + +// describeCluster returns a list of eks cluster +func (ek *EKSManager) describeCluster(nextToken *string, eksClusters []*eks.Cluster) ([]*eks.Cluster, error) { + input := &eks.ListClustersInput{ + NextToken: nextToken, + } + + resp, err := ek.client.ListClusters(input) + if err != nil { + log.WithField("error", err).Error("could not describe eks listclusters") + return nil, err + } + + if eksClusters == nil { + eksClusters = []*eks.Cluster{} + } + + for _, clusterName := range resp.Clusters { + clusterInput := &eks.DescribeClusterInput{ + Name: clusterName, + } + + resp, err := ek.client.DescribeCluster(clusterInput) + if err != nil { + log.WithField("error", err).Error("could not describe eks clusters") + return nil, err + } + + if resp.Cluster == nil { + log.WithFields(log.Fields{ + "clusterName": clusterName, + }).Error("Cluster with this name couldn't found") + } + + eksClusters = append(eksClusters, resp.Cluster) + } + + if resp.NextToken != nil { + return ek.describeCluster(resp.NextToken, eksClusters) + } + + return eksClusters, nil + +} diff --git a/collector/aws/resources/eks_test.go b/collector/aws/resources/eks_test.go new file mode 100644 index 00000000..020cb5db --- /dev/null +++ b/collector/aws/resources/eks_test.go @@ -0,0 +1,214 @@ +package resources + +import ( + "errors" + awsTestutils "finala/collector/aws/testutils" + "finala/collector/config" + collectorTestutils "finala/collector/testutils" + awsClient "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/eks" + "reflect" + "testing" + "time" +) + +var defaultListEKSMock = eks.ListClustersOutput{ + Clusters: []*string{ + awsClient.String("test"), + }, +} + +var defaultEKSClusterMock = eks.DescribeClusterOutput{ + Cluster: &eks.Cluster{ + Arn: awsClient.String("arn:aws:eks:us-west-2:012345678910:cluster/devel"), + CreatedAt: awsClient.Time(time.Now()), + Name: awsClient.String("test"), + Status: awsClient.String("ACTIVE"), + Version: awsClient.String("1.10"), + Endpoint: awsClient.String("https://EXAMPLE0A04F01705DD065655C30CC3D.yl4.us-west-2.eks.amazonaws.com"), + RoleArn: awsClient.String("arn:aws:iam::012345678910:role/eks-service-role-AWSServiceRoleForAmazonEKS-J7ONKE3BQ4PI"), + }, +} + +type MockAWSEKSClient struct { + responseListCluster eks.ListClustersOutput + responseDescribeCluster eks.DescribeClusterOutput + err error +} + +func (ek *MockAWSEKSClient) DescribeCluster(input *eks.DescribeClusterInput) (*eks.DescribeClusterOutput, error) { + return &ek.responseDescribeCluster, ek.err +} + +func (ek *MockAWSEKSClient) ListClusters(input *eks.ListClustersInput) (*eks.ListClustersOutput, error) { + return &ek.responseListCluster, ek.err +} + +func TestDescribeCluster(t *testing.T) { + + collector := collectorTestutils.NewMockCollector() + detector := awsTestutils.AWSManager(collector, nil, nil, "us-east-1") + + t.Run("valid", func(t *testing.T) { + + mockClient := MockAWSEKSClient{ + responseDescribeCluster: defaultEKSClusterMock, + responseListCluster: defaultListEKSMock, + } + + eksInterface, err := NewEKSManager(detector, &mockClient) + if err != nil { + t.Fatalf("unexpected eks error happened, got %v expected %v", err, nil) + } + eksManager, ok := eksInterface.(*EKSManager) + if !ok { + t.Fatalf("unexpected eks struct, got %s expected %s", reflect.TypeOf(eksManager), "*EKSManager") + } + + result, _ := eksManager.describeCluster(nil, nil) + + if len(result) != 1 { + t.Fatalf("unexpected eks clusters count, got %d expected %d", len(result), 1) + } + + }) + + t.Run("error", func(t *testing.T) { + mockClient := MockAWSEKSClient{ + responseListCluster: defaultListEKSMock, + responseDescribeCluster: defaultEKSClusterMock, + err: errors.New("error"), + } + + eksInterface, err := NewEKSManager(detector, &mockClient) + if err != nil { + t.Fatalf("unexpected eks error happened, got %v expected %v", err, nil) + } + eksManager, ok := eksInterface.(*EKSManager) + if !ok { + t.Fatalf("unexpected eks struct, got %s expected %s", reflect.TypeOf(eksManager), "*EKSManager") + } + + _, err = eksManager.describeCluster(nil, nil) + + if err == nil { + t.Fatalf("unexpected describe clusters error, returned empty") + } + }) +} + +func TestDetectEKS(t *testing.T) { + + metricConfig := []config.MetricConfig{ + { + Description: "test description write capacity", + Data: []config.MetricDataConfiguration{ + { + Name: "TestMetric", + Statistic: "Sum", + }, + }, + Constraint: config.MetricConstraintConfig{ + Operator: "==", + Value: 5, + }, + Period: 1, + StartTime: 1, + }, + } + + collector := collectorTestutils.NewMockCollector() + mockCloudwatch := awsTestutils.NewMockCloudwatch(nil) + mockPrice := awsTestutils.NewMockPricing(nil) + detector := awsTestutils.AWSManager(collector, mockCloudwatch, mockPrice, "us-east-1") + + mockClient := MockAWSEKSClient{ + responseDescribeCluster: defaultEKSClusterMock, + responseListCluster: defaultListEKSMock, + } + + eksInterface, err := NewEKSManager(detector, &mockClient) + if err != nil { + t.Fatalf("unexpected eks error happened, got %v expected %v", err, nil) + } + eksManager, ok := eksInterface.(*EKSManager) + if !ok { + t.Fatalf("unexpected eks struct, got %s expected %s", reflect.TypeOf(eksManager), "*EKSManager") + } + + response, _ := eksManager.Detect(metricConfig) + eksResponse, ok := response.([]DetectedEKS) + if !ok { + t.Fatalf("unexpected eks struct, got %s expected %s", reflect.TypeOf(response), "[]DetectedEKS") + } + + if len(eksResponse) != 1 { + t.Fatalf("unexpected eks detected, got %d expected %d", len(eksResponse), 1) + } + + if len(collector.Events) != 1 { + t.Fatalf("unexpected collector eks resources, got %d expected %d", len(collector.Events), 1) + } + + if len(collector.EventsCollectionStatus) != 2 { + t.Fatalf("unexpected resource status events count, got %d expected %d", len(collector.EventsCollectionStatus), 2) + } +} + +func TestDetectEKSError(t *testing.T) { + + metricConfig := []config.MetricConfig{ + { + Description: "test description write capacity", + Data: []config.MetricDataConfiguration{ + { + Name: "TestMetric", + Statistic: "Sum", + }, + }, + Constraint: config.MetricConstraintConfig{ + Operator: "==", + Value: 5, + }, + Period: 1, + StartTime: 1, + }, + } + + collector := collectorTestutils.NewMockCollector() + mockCloudwatch := awsTestutils.NewMockCloudwatch(nil) + mockPrice := awsTestutils.NewMockPricing(nil) + detector := awsTestutils.AWSManager(collector, mockCloudwatch, mockPrice, "us-east-1") + + mockClient := MockAWSEKSClient{ + err: errors.New(""), + } + + eksInterface, err := NewEKSManager(detector, &mockClient) + if err != nil { + t.Fatalf("unexpected eks error happened, got %v expected %v", err, nil) + } + eksManager, ok := eksInterface.(*EKSManager) + if !ok { + t.Fatalf("unexpected eks struct, got %s expected %s", reflect.TypeOf(eksManager), "*EKSManager") + } + + response, _ := eksManager.Detect(metricConfig) + eksResponse, ok := response.([]DetectedEKS) + if !ok { + t.Fatalf("unexpected eks struct, got %s expected %s", reflect.TypeOf(response), "[]DetectedEKS") + } + + if len(eksResponse) != 0 { + t.Fatalf("unexpected eks detected, got %d expected %d", len(eksResponse), 0) + } + + if len(collector.Events) != 0 { + t.Fatalf("unexpected collector eks resources, got %d expected %d", len(collector.Events), 0) + } + + if len(collector.EventsCollectionStatus) != 2 { + t.Fatalf("unexpected resource status events count, got %d expected %d", len(collector.EventsCollectionStatus), 2) + } + +} From 1165abe43d44b8999ee490c3868f5ca8694b93ab Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Mon, 21 Jun 2021 14:03:47 +0200 Subject: [PATCH 20/68] #SOS-25 change PricingFilter --- collector/aws/resources/eks.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/collector/aws/resources/eks.go b/collector/aws/resources/eks.go index 0af874a1..45a05fcb 100644 --- a/collector/aws/resources/eks.go +++ b/collector/aws/resources/eks.go @@ -89,7 +89,7 @@ func (ek *EKSManager) Detect(metrics []config.MetricConfig) (interface{}, error) for _, cluster := range clusters { log.WithField("name", *cluster.Name).Debug("checking eks") - price, _ := ek.awsManager.GetPricingClient().GetPrice(ek.getPricingFilterInput(cluster), "", ek.awsManager.GetRegion()) + price, _ := ek.awsManager.GetPricingClient().GetPrice(ek.getPricingFilterInput(), "", ek.awsManager.GetRegion()) for _, metric := range metrics { log.WithFields(log.Fields{ @@ -175,15 +175,25 @@ func (ek *EKSManager) Detect(metrics []config.MetricConfig) (interface{}, error) return detectedEKSClusters, nil } -func (ek *EKSManager) getPricingFilterInput(cluster *eks.Cluster) pricing.GetProductsInput { +func (ek *EKSManager) getPricingFilterInput() pricing.GetProductsInput { return pricing.GetProductsInput{ ServiceCode: &ek.servicePricingCode, Filters: []*pricing.Filter{ { Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("clusterName"), - Value: cluster.Name, + Field: awsClient.String("termType"), + Value: awsClient.String("OnDemand"), + }, + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("tenancy"), + Value: awsClient.String("Shared"), + }, + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("serviceCode"), + Value: awsClient.String("AmazonEKS"), }, }, } From 2effc76dd512a7eea097d190abc34445036c35f2 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Mon, 21 Jun 2021 15:10:11 +0200 Subject: [PATCH 21/68] #SOS-25 change PricingFilter --- collector/aws/resources/eks.go | 61 +++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/collector/aws/resources/eks.go b/collector/aws/resources/eks.go index 45a05fcb..54f3963f 100644 --- a/collector/aws/resources/eks.go +++ b/collector/aws/resources/eks.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "fmt" awsClient "github.com/aws/aws-sdk-go/aws" awsCloudwatch "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/eks" @@ -78,6 +79,15 @@ func (ek *EKSManager) Detect(metrics []config.MetricConfig) (interface{}, error) detectedEKSClusters := []DetectedEKS{} + pricingRegionPrefix, err := ek.awsManager.GetPricingClient().GetRegionPrefix(ek.awsManager.GetRegion()) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "region": ek.awsManager.GetRegion(), + }).Error("Could not get pricing region prefix") + ek.awsManager.GetCollector().CollectError(ek.Name, err) + return detectedEKSClusters, err + } + clusters, err := ek.describeCluster(nil, nil) if err != nil { ek.awsManager.GetCollector().CollectError(ek.Name, err) @@ -89,7 +99,13 @@ func (ek *EKSManager) Detect(metrics []config.MetricConfig) (interface{}, error) for _, cluster := range clusters { log.WithField("name", *cluster.Name).Debug("checking eks") - price, _ := ek.awsManager.GetPricingClient().GetPrice(ek.getPricingFilterInput(), "", ek.awsManager.GetRegion()) + price, _ := ek.awsManager.GetPricingClient().GetPrice(ek.getPricingFilterInput([]*pricing.Filter{ + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("usagetype"), + Value: awsClient.String(fmt.Sprintf("%sAmazonEKS-Hours:perCluster", pricingRegionPrefix)), + }, + }), "", ek.awsManager.GetRegion()) for _, metric := range metrics { log.WithFields(log.Fields{ @@ -175,29 +191,34 @@ func (ek *EKSManager) Detect(metrics []config.MetricConfig) (interface{}, error) return detectedEKSClusters, nil } -func (ek *EKSManager) getPricingFilterInput() pricing.GetProductsInput { +func (ek *EKSManager) getPricingFilterInput(extraFilters []*pricing.Filter) pricing.GetProductsInput { - return pricing.GetProductsInput{ - ServiceCode: &ek.servicePricingCode, - Filters: []*pricing.Filter{ - { - Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("termType"), - Value: awsClient.String("OnDemand"), - }, - { - Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("tenancy"), - Value: awsClient.String("Shared"), - }, - { - Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("serviceCode"), - Value: awsClient.String("AmazonEKS"), - }, + filters := []*pricing.Filter{ + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("termType"), + Value: awsClient.String("OnDemand"), + }, + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("tenancy"), + Value: awsClient.String("Shared"), }, + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("serviceCode"), + Value: awsClient.String("AmazonEKS"), + }, + } + + if extraFilters != nil { + filters = append(filters, extraFilters...) } + return pricing.GetProductsInput{ + ServiceCode: &ek.servicePricingCode, + Filters: filters, + } } // describeCluster returns a list of eks cluster From 99ad63695416a2b86475189bb92ca1091de98cfb Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Mon, 21 Jun 2021 16:16:12 +0200 Subject: [PATCH 22/68] #SOS-25 change PricingFilter --- collector/aws/resources/eks.go | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/collector/aws/resources/eks.go b/collector/aws/resources/eks.go index 54f3963f..8ad525ce 100644 --- a/collector/aws/resources/eks.go +++ b/collector/aws/resources/eks.go @@ -7,7 +7,6 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" - "fmt" awsClient "github.com/aws/aws-sdk-go/aws" awsCloudwatch "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/eks" @@ -79,15 +78,6 @@ func (ek *EKSManager) Detect(metrics []config.MetricConfig) (interface{}, error) detectedEKSClusters := []DetectedEKS{} - pricingRegionPrefix, err := ek.awsManager.GetPricingClient().GetRegionPrefix(ek.awsManager.GetRegion()) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "region": ek.awsManager.GetRegion(), - }).Error("Could not get pricing region prefix") - ek.awsManager.GetCollector().CollectError(ek.Name, err) - return detectedEKSClusters, err - } - clusters, err := ek.describeCluster(nil, nil) if err != nil { ek.awsManager.GetCollector().CollectError(ek.Name, err) @@ -103,7 +93,7 @@ func (ek *EKSManager) Detect(metrics []config.MetricConfig) (interface{}, error) { Type: awsClient.String("TERM_MATCH"), Field: awsClient.String("usagetype"), - Value: awsClient.String(fmt.Sprintf("%sAmazonEKS-Hours:perCluster", pricingRegionPrefix)), + Value: awsClient.String("USW2-AmazonEKS-Hours:perCluster"), }, }), "", ek.awsManager.GetRegion()) @@ -199,16 +189,6 @@ func (ek *EKSManager) getPricingFilterInput(extraFilters []*pricing.Filter) pric Field: awsClient.String("termType"), Value: awsClient.String("OnDemand"), }, - { - Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("tenancy"), - Value: awsClient.String("Shared"), - }, - { - Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("serviceCode"), - Value: awsClient.String("AmazonEKS"), - }, } if extraFilters != nil { From 12893e8d456db42daa75d97143e1880196c56e8f Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Mon, 21 Jun 2021 17:46:56 +0200 Subject: [PATCH 23/68] #SOS-25 change PricingFilter --- collector/aws/resources/eks.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/collector/aws/resources/eks.go b/collector/aws/resources/eks.go index 8ad525ce..f0200d3d 100644 --- a/collector/aws/resources/eks.go +++ b/collector/aws/resources/eks.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "fmt" awsClient "github.com/aws/aws-sdk-go/aws" awsCloudwatch "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/eks" @@ -78,6 +79,15 @@ func (ek *EKSManager) Detect(metrics []config.MetricConfig) (interface{}, error) detectedEKSClusters := []DetectedEKS{} + pricingRegionPrefix, err := ek.awsManager.GetPricingClient().GetRegionPrefix(ek.awsManager.GetRegion()) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "region": ek.awsManager.GetRegion(), + }).Error("Could not get pricing region prefix") + ek.awsManager.GetCollector().CollectError(ek.Name, err) + return detectedEKSClusters, err + } + clusters, err := ek.describeCluster(nil, nil) if err != nil { ek.awsManager.GetCollector().CollectError(ek.Name, err) @@ -93,7 +103,7 @@ func (ek *EKSManager) Detect(metrics []config.MetricConfig) (interface{}, error) { Type: awsClient.String("TERM_MATCH"), Field: awsClient.String("usagetype"), - Value: awsClient.String("USW2-AmazonEKS-Hours:perCluster"), + Value: awsClient.String(fmt.Sprintf("%sAmazonEKS-Hours:perCluster", pricingRegionPrefix)), }, }), "", ek.awsManager.GetRegion()) From 5b7187c956e03f422d5fb014f1d559e58268f1e1 Mon Sep 17 00:00:00 2001 From: DerOtt Date: Tue, 22 Jun 2021 13:26:35 +0200 Subject: [PATCH 24/68] Changed some things to test -> Delete if somehting breaks --- ui/package-lock.json | 2471 +++++++++++------ ui/package.json | 2 +- .../components/Dashboard/CSVDownloadButton.js | 37 + ui/src/components/Dashboard/Index.js | 2 + 4 files changed, 1663 insertions(+), 849 deletions(-) create mode 100644 ui/src/components/Dashboard/CSVDownloadButton.js diff --git a/ui/package-lock.json b/ui/package-lock.json index 07883fdc..4b863431 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -261,6 +261,12 @@ "@babel/types": "^7.7.4" } }, + "@babel/helper-validator-identifier": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.5.tgz", + "integrity": "sha512-5lsetuxCLilmVGyiLEfoHBRX8UCFD+1m2x3Rj97WrW3V7H3u4RWRXA4evMjImCsin2J2YT0QaVDGf+z8ondbAg==", + "dev": true + }, "@babel/helper-wrap-function": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.7.4.tgz", @@ -751,6 +757,45 @@ "@babel/helper-plugin-utils": "^7.0.0" } }, + "@babel/plugin-transform-runtime": { + "version": "7.11.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.11.0.tgz", + "integrity": "sha512-LFEsP+t3wkYBlis8w6/kmnd6Kb1dxTd+wGJ8MlxTGzQo//ehtqlVL4S9DNUa53+dtPSQobN2CXx4d81FqC58cw==", + "dev": true, + "requires": { + "@babel/helper-module-imports": "^7.10.4", + "@babel/helper-plugin-utils": "^7.10.4", + "resolve": "^1.8.1", + "semver": "^5.5.1" + }, + "dependencies": { + "@babel/helper-module-imports": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.14.5.tgz", + "integrity": "sha512-SwrNHu5QWS84XlHwGYPDtCxcA0hrSlL2yhWYLgeOc0w7ccOl2qv4s/nARI0aYZW+bSwAL5CukeXA47B/1NKcnQ==", + "dev": true, + "requires": { + "@babel/types": "^7.14.5" + } + }, + "@babel/helper-plugin-utils": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.14.5.tgz", + "integrity": "sha512-/37qQCE3K0vvZKwoK4XU/irIJQdIfCJuhU5eKnNxpFDsOkgFaUAwbv+RYw6eYgsC0E4hS7r5KqGULUogqui0fQ==", + "dev": true + }, + "@babel/types": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.14.5.tgz", + "integrity": "sha512-M/NzBpEL95I5Hh4dwhin5JlE7EzO5PHMAuzjxss3tiOBD46KfQvVedN/3jEPZvdRvtsK2222XfdHogNIttFgcg==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.14.5", + "to-fast-properties": "^2.0.0" + } + } + } + }, "@babel/plugin-transform-shorthand-properties": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.7.4.tgz", @@ -958,6 +1003,370 @@ "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.8.0.tgz", "integrity": "sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow==" }, + "@jimp/bmp": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.16.1.tgz", + "integrity": "sha512-iwyNYQeBawrdg/f24x3pQ5rEx+/GwjZcCXd3Kgc+ZUd+Ivia7sIqBsOnDaMZdKCBPlfW364ekexnlOqyVa0NWg==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1", + "bmp-js": "^0.1.0" + } + }, + "@jimp/core": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/core/-/core-0.16.1.tgz", + "integrity": "sha512-la7kQia31V6kQ4q1kI/uLimu8FXx7imWVajDGtwUG8fzePLWDFJyZl0fdIXVCL1JW2nBcRHidUot6jvlRDi2+g==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1", + "any-base": "^1.1.0", + "buffer": "^5.2.0", + "exif-parser": "^0.1.12", + "file-type": "^9.0.0", + "load-bmfont": "^1.3.1", + "mkdirp": "^0.5.1", + "phin": "^2.9.1", + "pixelmatch": "^4.0.2", + "tinycolor2": "^1.4.1" + }, + "dependencies": { + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + } + } + }, + "@jimp/custom": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.16.1.tgz", + "integrity": "sha512-DNUAHNSiUI/j9hmbatD6WN/EBIyeq4AO0frl5ETtt51VN1SvE4t4v83ZA/V6ikxEf3hxLju4tQ5Pc3zmZkN/3A==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/core": "^0.16.1" + } + }, + "@jimp/gif": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/gif/-/gif-0.16.1.tgz", + "integrity": "sha512-r/1+GzIW1D5zrP4tNrfW+3y4vqD935WBXSc8X/wm23QTY9aJO9Lw6PEdzpYCEY+SOklIFKaJYUAq/Nvgm/9ryw==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1", + "gifwrap": "^0.9.2", + "omggif": "^1.0.9" + } + }, + "@jimp/jpeg": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/jpeg/-/jpeg-0.16.1.tgz", + "integrity": "sha512-8352zrdlCCLFdZ/J+JjBslDvml+fS3Z8gttdml0We759PnnZGqrnPRhkOEOJbNUlE+dD4ckLeIe6NPxlS/7U+w==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1", + "jpeg-js": "0.4.2" + } + }, + "@jimp/plugin-blit": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.16.1.tgz", + "integrity": "sha512-fKFNARm32RoLSokJ8WZXHHH2CGzz6ire2n1Jh6u+XQLhk9TweT1DcLHIXwQMh8oR12KgjbgsMGvrMVlVknmOAg==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-blur": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.16.1.tgz", + "integrity": "sha512-1WhuLGGj9MypFKRcPvmW45ht7nXkOKu+lg3n2VBzIB7r4kKNVchuI59bXaCYQumOLEqVK7JdB4glaDAbCQCLyw==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-circle": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-circle/-/plugin-circle-0.16.1.tgz", + "integrity": "sha512-JK7yi1CIU7/XL8hdahjcbGA3V7c+F+Iw+mhMQhLEi7Q0tCnZ69YJBTamMiNg3fWPVfMuvWJJKOBRVpwNTuaZRg==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-color": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.16.1.tgz", + "integrity": "sha512-9yQttBAO5SEFj7S6nJK54f+1BnuBG4c28q+iyzm1JjtnehjqMg6Ljw4gCSDCvoCQ3jBSYHN66pmwTV74SU1B7A==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1", + "tinycolor2": "^1.4.1" + } + }, + "@jimp/plugin-contain": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-0.16.1.tgz", + "integrity": "sha512-44F3dUIjBDHN+Ym/vEfg+jtjMjAqd2uw9nssN67/n4FdpuZUVs7E7wadKY1RRNuJO+WgcD5aDQcsvurXMETQTg==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-cover": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-0.16.1.tgz", + "integrity": "sha512-YztWCIldBAVo0zxcQXR+a/uk3/TtYnpKU2CanOPJ7baIuDlWPsG+YE4xTsswZZc12H9Kl7CiziEbDtvF9kwA/Q==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-crop": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.16.1.tgz", + "integrity": "sha512-UQdva9oQzCVadkyo3T5Tv2CUZbf0klm2cD4cWMlASuTOYgaGaFHhT9st+kmfvXjKL8q3STkBu/zUPV6PbuV3ew==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-displace": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-0.16.1.tgz", + "integrity": "sha512-iVAWuz2+G6Heu8gVZksUz+4hQYpR4R0R/RtBzpWEl8ItBe7O6QjORAkhxzg+WdYLL2A/Yd4ekTpvK0/qW8hTVw==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-dither": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-0.16.1.tgz", + "integrity": "sha512-tADKVd+HDC9EhJRUDwMvzBXPz4GLoU6s5P7xkVq46tskExYSptgj5713J5Thj3NMgH9Rsqu22jNg1H/7tr3V9Q==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-fisheye": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-fisheye/-/plugin-fisheye-0.16.1.tgz", + "integrity": "sha512-BWHnc5hVobviTyIRHhIy9VxI1ACf4CeSuCfURB6JZm87YuyvgQh5aX5UDKtOz/3haMHXBLP61ZBxlNpMD8CG4A==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-flip": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-0.16.1.tgz", + "integrity": "sha512-KdxTf0zErfZ8DyHkImDTnQBuHby+a5YFdoKI/G3GpBl3qxLBvC+PWkS2F/iN3H7wszP7/TKxTEvWL927pypT0w==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-gaussian": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-gaussian/-/plugin-gaussian-0.16.1.tgz", + "integrity": "sha512-u9n4wjskh3N1mSqketbL6tVcLU2S5TEaFPR40K6TDv4phPLZALi1Of7reUmYpVm8mBDHt1I6kGhuCJiWvzfGyg==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-invert": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-invert/-/plugin-invert-0.16.1.tgz", + "integrity": "sha512-2DKuyVXANH8WDpW9NG+PYFbehzJfweZszFYyxcaewaPLN0GxvxVLOGOPP1NuUTcHkOdMFbE0nHDuB7f+sYF/2w==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-mask": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-0.16.1.tgz", + "integrity": "sha512-snfiqHlVuj4bSFS0v96vo2PpqCDMe4JB+O++sMo5jF5mvGcGL6AIeLo8cYqPNpdO6BZpBJ8MY5El0Veckhr39Q==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-normalize": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-normalize/-/plugin-normalize-0.16.1.tgz", + "integrity": "sha512-dOQfIOvGLKDKXPU8xXWzaUeB0nvkosHw6Xg1WhS1Z5Q0PazByhaxOQkSKgUryNN/H+X7UdbDvlyh/yHf3ITRaw==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-print": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-0.16.1.tgz", + "integrity": "sha512-ceWgYN40jbN4cWRxixym+csyVymvrryuKBQ+zoIvN5iE6OyS+2d7Mn4zlNgumSczb9GGyZZESIgVcBDA1ezq0Q==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1", + "load-bmfont": "^1.4.0" + } + }, + "@jimp/plugin-resize": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.16.1.tgz", + "integrity": "sha512-u4JBLdRI7dargC04p2Ha24kofQBk3vhaf0q8FwSYgnCRwxfvh2RxvhJZk9H7Q91JZp6wgjz/SjvEAYjGCEgAwQ==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-rotate": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.16.1.tgz", + "integrity": "sha512-ZUU415gDQ0VjYutmVgAYYxC9Og9ixu2jAGMCU54mSMfuIlmohYfwARQmI7h4QB84M76c9hVLdONWjuo+rip/zg==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-scale": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.16.1.tgz", + "integrity": "sha512-jM2QlgThIDIc4rcyughD5O7sOYezxdafg/2Xtd1csfK3z6fba3asxDwthqPZAgitrLgiKBDp6XfzC07Y/CefUw==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-shadow": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-shadow/-/plugin-shadow-0.16.1.tgz", + "integrity": "sha512-MeD2Is17oKzXLnsphAa1sDstTu6nxscugxAEk3ji0GV1FohCvpHBcec0nAq6/czg4WzqfDts+fcPfC79qWmqrA==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugin-threshold": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-threshold/-/plugin-threshold-0.16.1.tgz", + "integrity": "sha512-iGW8U/wiCSR0+6syrPioVGoSzQFt4Z91SsCRbgNKTAk7D+XQv6OI78jvvYg4o0c2FOlwGhqz147HZV5utoSLxA==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1" + } + }, + "@jimp/plugins": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/plugins/-/plugins-0.16.1.tgz", + "integrity": "sha512-c+lCqa25b+4q6mJZSetlxhMoYuiltyS+ValLzdwK/47+aYsq+kcJNl+TuxIEKf59yr9+5rkbpsPkZHLF/V7FFA==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/plugin-blit": "^0.16.1", + "@jimp/plugin-blur": "^0.16.1", + "@jimp/plugin-circle": "^0.16.1", + "@jimp/plugin-color": "^0.16.1", + "@jimp/plugin-contain": "^0.16.1", + "@jimp/plugin-cover": "^0.16.1", + "@jimp/plugin-crop": "^0.16.1", + "@jimp/plugin-displace": "^0.16.1", + "@jimp/plugin-dither": "^0.16.1", + "@jimp/plugin-fisheye": "^0.16.1", + "@jimp/plugin-flip": "^0.16.1", + "@jimp/plugin-gaussian": "^0.16.1", + "@jimp/plugin-invert": "^0.16.1", + "@jimp/plugin-mask": "^0.16.1", + "@jimp/plugin-normalize": "^0.16.1", + "@jimp/plugin-print": "^0.16.1", + "@jimp/plugin-resize": "^0.16.1", + "@jimp/plugin-rotate": "^0.16.1", + "@jimp/plugin-scale": "^0.16.1", + "@jimp/plugin-shadow": "^0.16.1", + "@jimp/plugin-threshold": "^0.16.1", + "timm": "^1.6.1" + } + }, + "@jimp/png": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/png/-/png-0.16.1.tgz", + "integrity": "sha512-iyWoCxEBTW0OUWWn6SveD4LePW89kO7ZOy5sCfYeDM/oTPLpR8iMIGvZpZUz1b8kvzFr27vPst4E5rJhGjwsdw==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/utils": "^0.16.1", + "pngjs": "^3.3.3" + } + }, + "@jimp/tiff": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/tiff/-/tiff-0.16.1.tgz", + "integrity": "sha512-3K3+xpJS79RmSkAvFMgqY5dhSB+/sxhwTFA9f4AVHUK0oKW+u6r52Z1L0tMXHnpbAdR9EJ+xaAl2D4x19XShkQ==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "utif": "^2.0.1" + } + }, + "@jimp/types": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/types/-/types-0.16.1.tgz", + "integrity": "sha512-g1w/+NfWqiVW4CaXSJyD28JQqZtm2eyKMWPhBBDCJN9nLCN12/Az0WFF3JUAktzdsEC2KRN2AqB1a2oMZBNgSQ==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/bmp": "^0.16.1", + "@jimp/gif": "^0.16.1", + "@jimp/jpeg": "^0.16.1", + "@jimp/png": "^0.16.1", + "@jimp/tiff": "^0.16.1", + "timm": "^1.6.1" + } + }, + "@jimp/utils": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-0.16.1.tgz", + "integrity": "sha512-8fULQjB0x4LzUSiSYG6ZtQl355sZjxbv8r9PPAuYHzS9sGiSHJQavNqK/nKnpDsVkU88/vRGcE7t3nMU0dEnVw==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "regenerator-runtime": "^0.13.3" + } + }, "@material-ui/core": { "version": "4.9.10", "resolved": "https://registry.npmjs.org/@material-ui/core/-/core-4.9.10.tgz", @@ -1184,12 +1593,6 @@ "resolved": "https://registry.npmjs.org/@react-dnd/shallowequal/-/shallowequal-2.0.0.tgz", "integrity": "sha512-Pc/AFTdwZwEKJxFJvlxrSmGe/di+aAOBn60sremrpLo6VI/6cmiUYNNwlI5KNYttg7uypzA3ILPMPgxB2GYZEg==" }, - "@sailshq/lodash": { - "version": "3.10.4", - "resolved": "https://registry.npmjs.org/@sailshq/lodash/-/lodash-3.10.4.tgz", - "integrity": "sha512-YXJqp9gdHcZKAmBY/WnwFpPtNQp2huD/ME2YMurH2YHJvxrVzYsmpKw/pb7yINArRpp8E++fwbQd3ajYXGA45Q==", - "dev": true - }, "@types/asap": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@types/asap/-/asap-2.0.0.tgz", @@ -1201,6 +1604,15 @@ "integrity": "sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==", "dev": true }, + "@types/favicons": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/@types/favicons/-/favicons-5.5.0.tgz", + "integrity": "sha512-s76OlRaBfqtGu2ZBobnZv2NETfqsQUVfKKlOkKNGo4ArBsqiblodKsnQ3j29hCCgmpQacEfLxealV96za+tzVQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/glob": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.1.tgz", @@ -1221,11 +1633,23 @@ "hoist-non-react-statics": "^3.3.0" } }, + "@types/html-minifier-terser": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@types/html-minifier-terser/-/html-minifier-terser-5.1.1.tgz", + "integrity": "sha512-giAlZwstKbmvMk1OO7WXSj4OZ0keXAcl2TQq4LWHiiPH2ByaH7WeUzng+Qej8UPxxv+8lRTuouo0iaNDBuzIBA==", + "dev": true + }, "@types/invariant": { "version": "2.2.33", "resolved": "https://registry.npmjs.org/@types/invariant/-/invariant-2.2.33.tgz", "integrity": "sha512-/jUNmS8d4bCKdqslfxW6dg/9Gksfzxz67IYfqApHn+HvHlMVXwYv2zpTDnS/yaK9BB0i0GlBTaYci0EFE62Hmw==" }, + "@types/json-schema": { + "version": "7.0.7", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.7.tgz", + "integrity": "sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==", + "dev": true + }, "@types/minimatch": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", @@ -1265,6 +1689,12 @@ "resolved": "https://registry.npmjs.org/@types/shallowequal/-/shallowequal-1.1.1.tgz", "integrity": "sha512-Lhni3aX80zbpdxRuWhnuYPm8j8UQaa571lHP/xI4W+7BAFhSIhRReXnqjEgT/XzPoXZTJkCqstFMJ8CZTK6IlQ==" }, + "@types/source-list-map": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@types/source-list-map/-/source-list-map-0.1.2.tgz", + "integrity": "sha512-K5K+yml8LTo9bWJI/rECfIPrGgxdpeNbj+d53lwN4QjW1MCwlkhUms+gtdzigTeUyBr09+u8BwOIY3MXvHdcsA==", + "dev": true + }, "@types/styled-jsx": { "version": "2.2.8", "resolved": "https://registry.npmjs.org/@types/styled-jsx/-/styled-jsx-2.2.8.tgz", @@ -1273,6 +1703,80 @@ "@types/react": "*" } }, + "@types/tapable": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.7.tgz", + "integrity": "sha512-0VBprVqfgFD7Ehb2vd8Lh9TG3jP98gvr8rgehQqzztZNI7o8zS8Ad4jyZneKELphpuE212D8J70LnSNQSyO6bQ==", + "dev": true + }, + "@types/uglify-js": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/@types/uglify-js/-/uglify-js-3.13.0.tgz", + "integrity": "sha512-EGkrJD5Uy+Pg0NUR8uA4bJ5WMfljyad0G+784vLCNUkD+QwOJXUbBYExXfVGf7YtyzdQp3L/XMYcliB987kL5Q==", + "dev": true, + "requires": { + "source-map": "^0.6.1" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "@types/webpack": { + "version": "4.41.29", + "resolved": "https://registry.npmjs.org/@types/webpack/-/webpack-4.41.29.tgz", + "integrity": "sha512-6pLaORaVNZxiB3FSHbyBiWM7QdazAWda1zvAq4SbZObZqHSDbWLi62iFdblVea6SK9eyBIVp5yHhKt/yNQdR7Q==", + "dev": true, + "requires": { + "@types/node": "*", + "@types/tapable": "^1", + "@types/uglify-js": "*", + "@types/webpack-sources": "*", + "anymatch": "^3.0.0", + "source-map": "^0.6.0" + }, + "dependencies": { + "anymatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", + "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", + "dev": true, + "requires": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "@types/webpack-sources": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/webpack-sources/-/webpack-sources-2.1.0.tgz", + "integrity": "sha512-LXn/oYIpBeucgP1EIJbKQ2/4ZmpvRl+dlrFdX7+94SKRUV3Evy3FsfMZY318vGhkWUS5MPhtOM3w1/hCOAOXcg==", + "dev": true, + "requires": { + "@types/node": "*", + "@types/source-list-map": "*", + "source-map": "^0.7.3" + }, + "dependencies": { + "source-map": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", + "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", + "dev": true + } + } + }, "@webassemblyjs/ast": { "version": "1.8.5", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.8.5.tgz", @@ -1557,6 +2061,12 @@ "color-convert": "^1.9.0" } }, + "any-base": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/any-base/-/any-base-1.1.0.tgz", + "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==", + "dev": true + }, "anymatch": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz", @@ -1782,15 +2292,6 @@ "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, - "async": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", - "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", - "dev": true, - "requires": { - "lodash": "^4.17.14" - } - }, "async-each": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz", @@ -1815,6 +2316,12 @@ "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" }, + "author-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/author-regex/-/author-regex-1.0.0.tgz", + "integrity": "sha1-0IiFvmubv5Q5/gh8dihyRfCoFFA=", + "dev": true + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -1990,6 +2497,55 @@ "file-uri-to-path": "1.0.0" } }, + "bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + }, + "dependencies": { + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "requires": { + "safe-buffer": "~5.2.0" + } + } + } + }, "block-stream": { "version": "0.0.9", "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.9.tgz", @@ -2006,9 +2562,9 @@ "dev": true }, "bmp-js": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.0.3.tgz", - "integrity": "sha1-ZBE+nHzxICs3btYHvzBibr5XsYo=", + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha1-4Fpj95amwf8l9Hcex62twUjAcjM=", "dev": true }, "bn.js": { @@ -2245,6 +2801,12 @@ "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", "dev": true }, + "buffer-json": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-json/-/buffer-json-2.0.0.tgz", + "integrity": "sha512-+jjPFVqyfF1esi9fvfUs3NqM0pH1ziZ36VP4hmA/y/Ssfo/5w5xHKfTw9BwQjoJ1w/oVtpLomqwUHKdefGyuHw==", + "dev": true + }, "buffer-xor": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", @@ -2325,6 +2887,126 @@ "unset-value": "^1.0.0" } }, + "cache-loader": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cache-loader/-/cache-loader-4.1.0.tgz", + "integrity": "sha512-ftOayxve0PwKzBF/GLsZNC9fJBXl8lkZE3TOsjkboHfVHVkL39iUEs1FO07A33mizmci5Dudt38UZrrYXDtbhw==", + "dev": true, + "requires": { + "buffer-json": "^2.0.0", + "find-cache-dir": "^3.0.0", + "loader-utils": "^1.2.3", + "mkdirp": "^0.5.1", + "neo-async": "^2.6.1", + "schema-utils": "^2.0.0" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "dev": true + }, + "fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "find-cache-dir": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.1.tgz", + "integrity": "sha512-t2GDMt3oGC/v+BMwzmllWDuJF/xcDtE5j/fCGbqDD7OLuJkj0cfh1YSA5VKPvwMeLFLNDBkwOKZ2X85jGLVftQ==", + "dev": true, + "requires": { + "commondir": "^1.0.1", + "make-dir": "^3.0.2", + "pkg-dir": "^4.1.0" + } + }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "requires": { + "p-locate": "^4.1.0" + } + }, + "make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "requires": { + "semver": "^6.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "requires": { + "p-limit": "^2.2.0" + } + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, + "pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "requires": { + "find-up": "^4.0.0" + } + }, + "schema-utils": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.1.tgz", + "integrity": "sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg==", + "dev": true, + "requires": { + "@types/json-schema": "^7.0.5", + "ajv": "^6.12.4", + "ajv-keywords": "^3.5.2" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, "callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -2332,13 +3014,21 @@ "dev": true }, "camel-case": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-3.0.0.tgz", - "integrity": "sha1-yjw2iKTpzzpM2nd9xNy8cTJJz3M=", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-4.1.2.tgz", + "integrity": "sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw==", "dev": true, "requires": { - "no-case": "^2.2.0", - "upper-case": "^1.1.1" + "pascal-case": "^3.1.2", + "tslib": "^2.0.3" + }, + "dependencies": { + "tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", + "dev": true + } } }, "camelcase": { @@ -2392,27 +3082,6 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, - "cheerio": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-0.19.0.tgz", - "integrity": "sha1-dy5wFfLuKZZQltcepBdbdas1SSU=", - "dev": true, - "requires": { - "css-select": "~1.0.0", - "dom-serializer": "~0.1.0", - "entities": "~1.1.1", - "htmlparser2": "~3.8.1", - "lodash": "^3.2.0" - }, - "dependencies": { - "lodash": { - "version": "3.10.1", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz", - "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=", - "dev": true - } - } - }, "chokidar": { "version": "2.1.8", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.1.8.tgz", @@ -2486,9 +3155,9 @@ "integrity": "sha512-JR/iSQOSt+LQIWwrwEzJ9uk0xfN3mTVYMwt1Ir5mUcSN6pU+V4zQFFaJsclJbPuAUQH+yfWef6tm7l1quW3C8Q==" }, "clean-css": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-4.2.1.tgz", - "integrity": "sha512-4ZxI6dy4lrY6FHzfiy1aEOXgu4LIsW2MhwG0VBKdcoGoH/XLFgaHSdLTGr4O8Be6A8r3MOphEiI8Gc1n0ecf3g==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-4.2.3.tgz", + "integrity": "sha512-VcMWDN54ZN/DS+g58HYL5/n4Zrqe8vHJpGA8KdgUXFU4fuP/aHNw8eld9SyEIyabIMJX/0RaY/fplOo5hYLSFA==", "dev": true, "requires": { "source-map": "~0.6.0" @@ -2526,58 +3195,16 @@ "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=", "dev": true }, - "cliui": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", - "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=", - "dev": true, - "requires": { - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wrap-ansi": "^2.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "dev": true, - "requires": { - "number-is-nan": "^1.0.0" - } - }, - "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "dev": true, - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "dev": true, - "requires": { - "ansi-regex": "^2.0.0" - } - } - } - }, "clone": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", - "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=", + "dev": true + }, + "clone-buffer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/clone-buffer/-/clone-buffer-1.0.0.tgz", + "integrity": "sha1-4+JbIHrE5wGvch4staFnksrD3Fg=", "dev": true }, "clone-deep": { @@ -2593,11 +3220,54 @@ } }, "clone-stats": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/clone-stats/-/clone-stats-0.0.1.tgz", - "integrity": "sha1-uI+UqCzzi4eR1YBG6kAprYjKmdE=", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/clone-stats/-/clone-stats-1.0.0.tgz", + "integrity": "sha1-s3gt/4u1R04Yuba/D9/ngvh3doA=", "dev": true }, + "cloneable-readable": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/cloneable-readable/-/cloneable-readable-1.1.3.tgz", + "integrity": "sha512-2EF8zTQOxYq70Y4XKtorQupqF0m49MBz2/yf5Bj+MHjvpG3Hy7sImifnqD6UA+TKYxeSV+u6qqQPawN5UvnpKQ==", + "dev": true, + "requires": { + "inherits": "^2.0.1", + "process-nextick-args": "^2.0.0", + "readable-stream": "^2.3.5" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dev": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "clsx": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/clsx/-/clsx-1.1.0.tgz", @@ -2618,6 +3288,16 @@ "object-visit": "^1.0.0" } }, + "color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/color/-/color-3.1.3.tgz", + "integrity": "sha512-xgXAcTHa2HeFCGLE9Xs/R82hujGtu9Jd9x4NW3T34+OMs7VoPsjwzRczKHvTAHeJwWFwX5j15+MgAppE8ztObQ==", + "dev": true, + "requires": { + "color-convert": "^1.9.1", + "color-string": "^1.5.4" + } + }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -2631,6 +3311,16 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" }, + "color-string": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.5.tgz", + "integrity": "sha512-jgIoum0OfQfq9Whcfc2z/VhCNcmQjWbey6qBX0vqt7YICflUmBCh9E9CiQD5GSJ+Uehixm3NUwHVhqUAWRivZg==", + "dev": true, + "requires": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "colors": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", @@ -2647,9 +3337,9 @@ } }, "commander": { - "version": "2.17.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz", - "integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", "dev": true }, "commondir": { @@ -2945,15 +3635,16 @@ } }, "css-select": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.0.0.tgz", - "integrity": "sha1-sRIcpRhI3SZOIkTQWM7iVN7rRLA=", + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.1.3.tgz", + "integrity": "sha512-gT3wBNd9Nj49rAbmtFHj1cljIAOLYSX1nZ8CB7TBO3INYckygm5B7LISU/szY//YmdiSLbJvDLOx9VnMVpMBxA==", "dev": true, "requires": { - "boolbase": "~1.0.0", - "css-what": "1.0", - "domutils": "1.4", - "nth-check": "~1.0.0" + "boolbase": "^1.0.0", + "css-what": "^5.0.0", + "domhandler": "^4.2.0", + "domutils": "^2.6.0", + "nth-check": "^2.0.0" } }, "css-vendor": { @@ -2981,9 +3672,9 @@ } }, "css-what": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-1.0.0.tgz", - "integrity": "sha1-18wt9FGAZm+Z0rFEYmOUaeAPc2w=", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-5.0.1.tgz", + "integrity": "sha512-FYDTSHb/7KXsWICVsxdmiExPjCfRC4qRFBdVwv7Ax9hMnvMmEjP9RfxTEZ3qPZGmADDn2vAKSo9UcN1jKVYscg==", "dev": true }, "cssesc": { @@ -3050,6 +3741,21 @@ "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=" }, + "decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "dev": true, + "requires": { + "mimic-response": "^2.0.0" + } + }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", @@ -3156,6 +3862,12 @@ "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz", "integrity": "sha1-8NZtA2cqglyxtzvbP+YjEMjlUrc=" }, + "detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", + "dev": true + }, "diffie-hellman": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", @@ -3231,13 +3943,14 @@ } }, "dom-serializer": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", - "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.3.2.tgz", + "integrity": "sha512-5c54Bk5Dw4qAxNOI1pFEizPSjVsx5+bpJKmL2kPn8JhBUq2q09tTCa3mjijun2NfK78NMouDYNMBkOrPZiS+ig==", "dev": true, "requires": { - "domelementtype": "^1.3.0", - "entities": "^1.1.1" + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" } }, "dom-walk": { @@ -3253,27 +3966,47 @@ "dev": true }, "domelementtype": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", - "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.2.0.tgz", + "integrity": "sha512-DtBMo82pv1dFtUmHyr48beiuq792Sxohr+8Hm9zoxklYPfa6n0Z3Byjj2IV7bmr2IyqClnqEQhfgHJJ5QF0R5A==", "dev": true }, "domhandler": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.3.0.tgz", - "integrity": "sha1-LeWaCCLVAn+r/28DLCsloqir5zg=", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.2.0.tgz", + "integrity": "sha512-zk7sgt970kzPks2Bf+dwT/PLzghLnsivb9CcxkvR8Mzr66Olr0Ofd8neSbglHJHaHa2MadfoSdNlKYAaafmWfA==", "dev": true, "requires": { - "domelementtype": "1" + "domelementtype": "^2.2.0" } }, "domutils": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.4.3.tgz", - "integrity": "sha1-CGVRN5bGswYDGFDhdVFrr4C3Km8=", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.7.0.tgz", + "integrity": "sha512-8eaHa17IwJUPAiB+SoTYBo5mCdeMgdcAoXJ59m6DT1vw+5iLS3gNoqYaRowaBKtGVrOF1Jz4yDTgYKLK2kvfJg==", + "dev": true, + "requires": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + } + }, + "dot-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.4.tgz", + "integrity": "sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==", "dev": true, "requires": { - "domelementtype": "1" + "no-case": "^3.0.4", + "tslib": "^2.0.3" + }, + "dependencies": { + "tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", + "dev": true + } } }, "duplexify": { @@ -3443,9 +4176,9 @@ } }, "entities": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", - "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", "dev": true }, "errno": { @@ -3791,6 +4524,12 @@ } } }, + "expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "dev": true + }, "expand-tilde": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz", @@ -3955,74 +4694,6 @@ } } }, - "extract-text-webpack-plugin": { - "version": "4.0.0-beta.0", - "resolved": "https://registry.npmjs.org/extract-text-webpack-plugin/-/extract-text-webpack-plugin-4.0.0-beta.0.tgz", - "integrity": "sha512-Hypkn9jUTnFr0DpekNam53X47tXn3ucY08BQumv7kdGgeVUBLq3DJHJTi6HNxv4jl9W+Skxjz9+RnK0sJyqqjA==", - "dev": true, - "requires": { - "async": "^2.4.1", - "loader-utils": "^1.1.0", - "schema-utils": "^0.4.5", - "webpack-sources": "^1.1.0" - }, - "dependencies": { - "schema-utils": { - "version": "0.4.7", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.7.tgz", - "integrity": "sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==", - "dev": true, - "requires": { - "ajv": "^6.1.0", - "ajv-keywords": "^3.1.0" - } - } - } - }, - "extract-zip": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz", - "integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=", - "dev": true, - "requires": { - "concat-stream": "1.6.2", - "debug": "2.6.9", - "mkdirp": "0.5.1", - "yauzl": "2.4.1" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - }, - "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", - "dev": true - }, - "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "dev": true, - "requires": { - "minimist": "0.0.8" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", - "dev": true - } - } - }, "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", @@ -4053,73 +4724,150 @@ "dev": true }, "favicons": { - "version": "4.8.6", - "resolved": "https://registry.npmjs.org/favicons/-/favicons-4.8.6.tgz", - "integrity": "sha1-orE4AKs/7CcVvI8n+oQdA41HYeI=", - "dev": true, - "requires": { - "async": "^1.5.0", - "cheerio": "^0.19.0", - "clone": "^1.0.2", - "colors": "^1.1.2", - "harmony-reflect": "^1.4.2", - "image-size": "^0.4.0", - "jimp": "^0.2.13", - "jsontoxml": "0.0.11", - "merge-defaults": "^0.2.1", - "mkdirp": "^0.5.1", - "node-rest-client": "^1.5.1", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/favicons/-/favicons-6.2.2.tgz", + "integrity": "sha512-qhvFqbhlXA/JYIDYuxTrE4uT9rcpTCrWvF3UG0GxBoLl/XgFBBTrZkQvASrkMebSwDCJ9kKGypRWIbvoRZLBsw==", + "dev": true, + "requires": { + "clone": "^2.1.2", + "colors": "^1.4.0", + "image-size": "^0.8.3", + "jimp": "^0.16.1", + "jsontoxml": "^1.0.1", + "lodash.defaultsdeep": "^4.6.1", "require-directory": "^2.1.1", - "svg2png": "~3.0.1", - "through2": "^2.0.0", - "tinycolor2": "^1.1.2", - "to-ico": "^1.1.2", - "underscore": "^1.8.3", - "vinyl": "^1.1.0" - }, - "dependencies": { - "async": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", - "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", + "sharp": "^0.28.2", + "through2": "^4.0.2", + "tinycolor2": "^1.4.2", + "to-ico": "^1.1.5", + "vinyl": "^2.2.1", + "xml2js": "^0.4.23" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", "dev": true + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "requires": { + "safe-buffer": "~5.2.0" + } + }, + "through2": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "dev": true, + "requires": { + "readable-stream": "3" + } } } }, "favicons-webpack-plugin": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/favicons-webpack-plugin/-/favicons-webpack-plugin-0.0.9.tgz", - "integrity": "sha1-32PoDFVrgE5JJeyOBb7jY5FXPck=", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/favicons-webpack-plugin/-/favicons-webpack-plugin-4.2.0.tgz", + "integrity": "sha512-rLbQ3yoe7iOJe7rVh8hABwPk1qFj+F68u+7LgZTfpERRFd61Fc0ou4lP09naDvM4eexIsnQGXtsiNaLY97nzDQ==", "dev": true, "requires": { - "favicons": "^4.8.3", - "loader-utils": "^0.2.14", - "lodash": "^4.11.1" + "@types/favicons": "5.5.0", + "cache-loader": "^4.1.0", + "camelcase": "^5.3.1", + "favicons": "^6.2.0", + "find-cache-dir": "^3.2.0", + "find-root": "^1.1.0", + "html-webpack-plugin": ">=4.0.0 || ^4.0.0-beta.11", + "loader-utils": "^1.2.3", + "parse-author": "^2.0.0", + "parse5": "^5.1.0", + "tapable": "^1.1.3" }, "dependencies": { - "big.js": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/big.js/-/big.js-3.2.0.tgz", - "integrity": "sha512-+hN/Zh2D08Mx65pZ/4g5bsmNiZUuChDiQfTUQ7qJr4/kuopCr88xZsAXv6mBoZEsUI4OuGHlX59qE94K2mMW8Q==", - "dev": true + "find-cache-dir": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.1.tgz", + "integrity": "sha512-t2GDMt3oGC/v+BMwzmllWDuJF/xcDtE5j/fCGbqDD7OLuJkj0cfh1YSA5VKPvwMeLFLNDBkwOKZ2X85jGLVftQ==", + "dev": true, + "requires": { + "commondir": "^1.0.1", + "make-dir": "^3.0.2", + "pkg-dir": "^4.1.0" + } }, - "json5": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-0.5.1.tgz", - "integrity": "sha1-Hq3nrMASA0rYTiOWdn6tn6VJWCE=", + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "requires": { + "p-locate": "^4.1.0" + } + }, + "make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "requires": { + "semver": "^6.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "requires": { + "p-limit": "^2.2.0" + } + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true }, - "loader-utils": { - "version": "0.2.17", - "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-0.2.17.tgz", - "integrity": "sha1-+G5jdNQyBabmxg6RlvF8Apm/s0g=", + "pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", "dev": true, "requires": { - "big.js": "^3.1.3", - "emojis-list": "^2.0.0", - "json5": "^0.5.0", - "object-assign": "^4.0.1" + "find-up": "^4.0.0" } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true } } }, @@ -4144,15 +4892,6 @@ } } }, - "fd-slicer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz", - "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=", - "dev": true, - "requires": { - "pend": "~1.2.0" - } - }, "figgy-pudding": { "version": "3.5.1", "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.1.tgz", @@ -4178,9 +4917,9 @@ } }, "file-type": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", - "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek=", + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-9.0.0.tgz", + "integrity": "sha512-Qe/5NJrgIOlwijpq3B7BEpzPFcgzggOTagZmkXQY4LA6bsXKTUstK7Wp12lEJ/mLKTpvIZxmIuRcLYWT6ov9lw==", "dev": true }, "file-uri-to-path": { @@ -4277,6 +5016,12 @@ "pkg-dir": "^3.0.0" } }, + "find-root": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/find-root/-/find-root-1.1.0.tgz", + "integrity": "sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==", + "dev": true + }, "find-up": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", @@ -4473,16 +5218,11 @@ "react-dom": "^16.3.0" } }, - "fs-extra": { + "fs-constants": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", - "integrity": "sha1-zTzl9+fLYUWIP8rjGR6Yd/hYeVA=", - "dev": true, - "requires": { - "graceful-fs": "^4.1.2", - "jsonfile": "^2.1.0", - "klaw": "^1.0.0" - } + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true }, "fs-write-stream-atomic": { "version": "1.0.10", @@ -5248,6 +5988,22 @@ "assert-plus": "^1.0.0" } }, + "gifwrap": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/gifwrap/-/gifwrap-0.9.2.tgz", + "integrity": "sha512-fcIswrPaiCDAyO8xnWvHSZdWChjKXUanKKpAiWWJ/UTkEi/aYKn5+90e7DE820zbEaVR9CE2y4z9bzhQijZ0BA==", + "dev": true, + "requires": { + "image-q": "^1.1.1", + "omggif": "^1.0.10" + } + }, + "github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=", + "dev": true + }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -5391,12 +6147,6 @@ "har-schema": "^2.0.0" } }, - "harmony-reflect": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/harmony-reflect/-/harmony-reflect-1.6.1.tgz", - "integrity": "sha512-WJTeyp0JzGtHcuMsi7rw2VwtkvLa+JyfEKJCFyfcS0+CDkjQ5lHPu7zEhFZP+PDSRrEgXa5Ah0l1MbgbE41XjA==", - "dev": true - }, "has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -5489,16 +6239,6 @@ "minimalistic-assert": "^1.0.1" } }, - "hasha": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/hasha/-/hasha-2.2.0.tgz", - "integrity": "sha1-eNfL/B5tZjA/55g3NlmEUXsvbuE=", - "dev": true, - "requires": { - "is-stream": "^1.0.1", - "pinkie-promise": "^2.0.0" - } - }, "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -5557,91 +6297,75 @@ "integrity": "sha1-DfKTUfByEWNRXfueVUPl9u7VFi8=", "dev": true }, - "html-minifier": { - "version": "3.5.21", - "resolved": "https://registry.npmjs.org/html-minifier/-/html-minifier-3.5.21.tgz", - "integrity": "sha512-LKUKwuJDhxNa3uf/LPR/KVjm/l3rBqtYeCOAekvG8F1vItxMUpueGd94i/asDDr8/1u7InxzFA5EeGjhhG5mMA==", - "dev": true, - "requires": { - "camel-case": "3.0.x", - "clean-css": "4.2.x", - "commander": "2.17.x", - "he": "1.2.x", - "param-case": "2.1.x", - "relateurl": "0.2.x", - "uglify-js": "3.4.x" - } - }, - "html-webpack-plugin": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", - "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=", + "html-minifier-terser": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/html-minifier-terser/-/html-minifier-terser-5.1.1.tgz", + "integrity": "sha512-ZPr5MNObqnV/T9akshPKbVgyOqLmy+Bxo7juKCfTfnjNniTAMdy4hz21YQqoofMBJD2kdREaqPPdThoR78Tgxg==", "dev": true, "requires": { - "html-minifier": "^3.2.3", - "loader-utils": "^0.2.16", - "lodash": "^4.17.3", - "pretty-error": "^2.0.2", - "tapable": "^1.0.0", - "toposort": "^1.0.0", - "util.promisify": "1.0.0" + "camel-case": "^4.1.1", + "clean-css": "^4.2.3", + "commander": "^4.1.1", + "he": "^1.2.0", + "param-case": "^3.0.3", + "relateurl": "^0.2.7", + "terser": "^4.6.3" }, "dependencies": { - "big.js": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/big.js/-/big.js-3.2.0.tgz", - "integrity": "sha512-+hN/Zh2D08Mx65pZ/4g5bsmNiZUuChDiQfTUQ7qJr4/kuopCr88xZsAXv6mBoZEsUI4OuGHlX59qE94K2mMW8Q==", - "dev": true - }, - "json5": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-0.5.1.tgz", - "integrity": "sha1-Hq3nrMASA0rYTiOWdn6tn6VJWCE=", + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "dev": true }, - "loader-utils": { - "version": "0.2.17", - "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-0.2.17.tgz", - "integrity": "sha1-+G5jdNQyBabmxg6RlvF8Apm/s0g=", + "terser": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-4.8.0.tgz", + "integrity": "sha512-EAPipTNeWsb/3wLPeup1tVPaXfIaU68xMnVdPafIL1TV05OhASArYyIfFvnvJCNrR2NIOvDVNNTFRa+Re2MWyw==", "dev": true, "requires": { - "big.js": "^3.1.3", - "emojis-list": "^2.0.0", - "json5": "^0.5.0", - "object-assign": "^4.0.1" + "commander": "^2.20.0", + "source-map": "~0.6.1", + "source-map-support": "~0.5.12" + }, + "dependencies": { + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true + } } } } }, + "html-webpack-plugin": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-4.3.0.tgz", + "integrity": "sha512-C0fzKN8yQoVLTelcJxZfJCE+aAvQiY2VUf3UuKrR4a9k5UMWYOtpDLsaXwATbcVCnI05hUS7L9ULQHWLZhyi3w==", + "dev": true, + "requires": { + "@types/html-minifier-terser": "^5.0.0", + "@types/tapable": "^1.0.5", + "@types/webpack": "^4.41.8", + "html-minifier-terser": "^5.0.1", + "loader-utils": "^1.2.3", + "lodash": "^4.17.15", + "pretty-error": "^2.1.1", + "tapable": "^1.1.3", + "util.promisify": "1.0.0" + } + }, "htmlparser2": { - "version": "3.8.3", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", - "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-6.1.0.tgz", + "integrity": "sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==", "dev": true, "requires": { - "domelementtype": "1", - "domhandler": "2.3", - "domutils": "1.5", - "entities": "1.0", - "readable-stream": "1.1" - }, - "dependencies": { - "domutils": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", - "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", - "dev": true, - "requires": { - "dom-serializer": "0", - "domelementtype": "1" - } - }, - "entities": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-1.0.0.tgz", - "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY=", - "dev": true - } + "domelementtype": "^2.0.1", + "domhandler": "^4.0.0", + "domutils": "^2.5.2", + "entities": "^2.0.0" } }, "http-errors": { @@ -5727,12 +6451,21 @@ "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", "dev": true }, - "image-size": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.4.0.tgz", - "integrity": "sha1-1LTh9hlS5MvBzqmmsMkV/stwdRA=", + "image-q": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/image-q/-/image-q-1.1.1.tgz", + "integrity": "sha1-/IQJlmRGC5DKhi2TALa/u7+/gFY=", "dev": true }, + "image-size": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.8.3.tgz", + "integrity": "sha512-SMtq1AJ+aqHB45c3FsB4ERK0UCiA2d3H1uq8s+8T0Pf8A3W4teyBQyaFaktH6xvZqh+npwlKU7i4fJo0r7TYTg==", + "dev": true, + "requires": { + "queue": "6.0.1" + } + }, "immutable": { "version": "3.8.2", "resolved": "https://registry.npmjs.org/immutable/-/immutable-3.8.2.tgz", @@ -5862,12 +6595,6 @@ "loose-envify": "^1.0.0" } }, - "invert-kv": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", - "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=", - "dev": true - }, "ip-regex": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-1.0.3.tgz", @@ -5988,9 +6715,9 @@ "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" }, "is-function": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-function/-/is-function-1.0.1.tgz", - "integrity": "sha1-Es+5i2W1fdPRk6MSH19uL0N2ArU=", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-function/-/is-function-1.0.2.tgz", + "integrity": "sha512-lw7DUp0aWXYg+CBCN+JKkcE0Q2RayZnSvnZBlwgxHBQhqt5pZNVy4Ri7H9GmmXkdu7LUthszM+Tor1u/2iBcpQ==", "dev": true }, "is-glob": { @@ -6048,6 +6775,12 @@ "path-is-inside": "^1.0.2" } }, + "is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=", + "dev": true + }, "is-plain-object": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", @@ -6139,50 +6872,22 @@ "dev": true }, "jimp": { - "version": "0.2.28", - "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.2.28.tgz", - "integrity": "sha1-3VKak3GQ9ClXp5N9Gsw6d2KZbqI=", - "dev": true, - "requires": { - "bignumber.js": "^2.1.0", - "bmp-js": "0.0.3", - "es6-promise": "^3.0.2", - "exif-parser": "^0.1.9", - "file-type": "^3.1.0", - "jpeg-js": "^0.2.0", - "load-bmfont": "^1.2.3", - "mime": "^1.3.4", - "mkdirp": "0.5.1", - "pixelmatch": "^4.0.0", - "pngjs": "^3.0.0", - "read-chunk": "^1.0.1", - "request": "^2.65.0", - "stream-to-buffer": "^0.1.0", - "tinycolor2": "^1.1.2", - "url-regex": "^3.0.0" - }, - "dependencies": { - "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", - "dev": true - }, - "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "dev": true, - "requires": { - "minimist": "0.0.8" - } - } + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.16.1.tgz", + "integrity": "sha512-+EKVxbR36Td7Hfd23wKGIeEyHbxShZDX6L8uJkgVW3ESA9GiTEPK08tG1XI2r/0w5Ch0HyJF5kPqF9K7EmGjaw==", + "dev": true, + "requires": { + "@babel/runtime": "^7.7.2", + "@jimp/custom": "^0.16.1", + "@jimp/plugins": "^0.16.1", + "@jimp/types": "^0.16.1", + "regenerator-runtime": "^0.13.3" } }, "jpeg-js": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.2.0.tgz", - "integrity": "sha1-U+RI7J0mPmgyZkZ+lELSxaLvVII=", + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.2.tgz", + "integrity": "sha512-+az2gi/hvex7eLTMTlbRLOhH6P6WFdk2ITI8HJsaH2VqYO0I594zXSYEP+tf4FW+8Cy68ScDXoAsQdyQanv3sw==", "dev": true }, "js-base64": { @@ -6263,19 +6968,10 @@ "minimist": "^1.2.0" } }, - "jsonfile": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", - "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", - "dev": true, - "requires": { - "graceful-fs": "^4.1.6" - } - }, "jsontoxml": { - "version": "0.0.11", - "resolved": "https://registry.npmjs.org/jsontoxml/-/jsontoxml-0.0.11.tgz", - "integrity": "sha1-Nzq1sgcL43N6X7PjL9G3uBhwyqQ=", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/jsontoxml/-/jsontoxml-1.0.1.tgz", + "integrity": "sha512-dtKGq0K8EWQBRqcAaePSgKR4Hyjfsz/LkurHSV3Cxk4H+h2fWDeaN2jzABz+ZmOJylgXS7FGeWmbZ6jgYUMdJQ==", "dev": true }, "jsprim": { @@ -6377,35 +7073,11 @@ "object.assign": "^4.1.0" } }, - "kew": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/kew/-/kew-0.7.0.tgz", - "integrity": "sha1-edk9LTM2PW/dKXCzNdkUGtWR15s=", - "dev": true - }, "kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" }, - "klaw": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz", - "integrity": "sha1-QIhDO0azsbolnXh4XY6W9zugJDk=", - "dev": true, - "requires": { - "graceful-fs": "^4.1.9" - } - }, - "lcid": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", - "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=", - "dev": true, - "requires": { - "invert-kv": "^1.0.0" - } - }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", @@ -6417,9 +7089,9 @@ } }, "load-bmfont": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.0.tgz", - "integrity": "sha512-kT63aTAlNhZARowaNYcY29Fn/QYkc52M3l6V1ifRcPewg2lvUZDAj7R6dXjOL9D0sict76op3T5+odumDSF81g==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz", + "integrity": "sha512-8UyQoYmdRDy81Brz6aLAUhfZLwr5zV0L3taTQ4hju7m6biuwiWiJXjPhBJxbUQJA8PrkvJ/7Enqmwk2sM14soA==", "dev": true, "requires": { "buffer-equal": "0.0.1", @@ -6559,6 +7231,12 @@ "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=" }, + "lodash.defaultsdeep": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/lodash.defaultsdeep/-/lodash.defaultsdeep-4.6.1.tgz", + "integrity": "sha512-3j8wdDzYuWO3lM3Reg03MuQR957t287Rpcxp1njpEa8oDrikb+FwGdW3n+FELh/A6qib6yPit0j/pv9G/yeAqA==", + "dev": true + }, "lodash.find": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.find/-/lodash.find-4.6.0.tgz", @@ -6619,10 +7297,21 @@ } }, "lower-case": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-1.1.4.tgz", - "integrity": "sha1-miyr0bno4K6ZOkv31YdcOcQujqw=", - "dev": true + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", + "integrity": "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==", + "dev": true, + "requires": { + "tslib": "^2.0.3" + }, + "dependencies": { + "tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", + "dev": true + } + } }, "lru-cache": { "version": "4.1.5", @@ -6789,15 +7478,6 @@ "trim-newlines": "^1.0.0" } }, - "merge-defaults": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/merge-defaults/-/merge-defaults-0.2.2.tgz", - "integrity": "sha512-rKkxPFgGDZfmen0IN8BKRsGEbFU3PdO0RhR1GjOk+BLJF7+LAIhs5bUG3s26FkbB5bfIn9il25KkntRGdqHQ3A==", - "dev": true, - "requires": { - "@sailshq/lodash": "^3.10.2" - } - }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -6867,6 +7547,12 @@ "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", "dev": true }, + "mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", + "dev": true + }, "min-document": { "version": "2.19.0", "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz", @@ -6876,6 +7562,18 @@ "dom-walk": "^0.1.0" } }, + "mini-css-extract-plugin": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-0.10.0.tgz", + "integrity": "sha512-QgKgJBjaJhxVPwrLNqqwNS0AGkuQQ31Hp4xGXEK/P7wehEg6qmNtReHKai3zRXqY60wGVWLYcOMJK2b98aGc3A==", + "dev": true, + "requires": { + "loader-utils": "^1.1.0", + "normalize-url": "1.9.1", + "schema-utils": "^1.0.0", + "webpack-sources": "^1.1.0" + } + }, "minimalistic-assert": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", @@ -6966,6 +7664,12 @@ "minimist": "^1.2.5" } }, + "mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true + }, "moment": { "version": "2.25.3", "resolved": "https://registry.npmjs.org/moment/-/moment-2.25.3.tgz", @@ -7044,6 +7748,12 @@ "to-regex": "^3.0.1" } }, + "napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", + "dev": true + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -7068,14 +7778,38 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" }, "no-case": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/no-case/-/no-case-2.3.2.tgz", - "integrity": "sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz", + "integrity": "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==", + "dev": true, + "requires": { + "lower-case": "^2.0.2", + "tslib": "^2.0.3" + }, + "dependencies": { + "tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", + "dev": true + } + } + }, + "node-abi": { + "version": "2.30.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.30.0.tgz", + "integrity": "sha512-g6bZh3YCKQRdwuO/tSZZYJAw622SjsRfJ2X0Iy4sSOHZ34/sPPdVBn8fev2tj7njzLwuqPw9uMtGsGkO5kIQvg==", "dev": true, "requires": { - "lower-case": "^1.1.1" + "semver": "^5.4.1" } }, + "node-addon-api": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz", + "integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==", + "dev": true + }, "node-fetch": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", @@ -7224,33 +7958,6 @@ } } }, - "node-rest-client": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/node-rest-client/-/node-rest-client-1.8.0.tgz", - "integrity": "sha1-jTxWa4F+JzlMtyc3g6Qcrv4+WVU=", - "dev": true, - "requires": { - "debug": "~2.2.0", - "xml2js": ">=0.2.4" - }, - "dependencies": { - "debug": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", - "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=", - "dev": true, - "requires": { - "ms": "0.7.1" - } - }, - "ms": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz", - "integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg=", - "dev": true - } - } - }, "node-sass": { "version": "4.14.1", "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-4.14.1.tgz", @@ -7355,6 +8062,18 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, + "normalize-url": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-1.9.1.tgz", + "integrity": "sha1-LMDWazHqIwNkWENuNiDYWVTGbDw=", + "dev": true, + "requires": { + "object-assign": "^4.0.1", + "prepend-http": "^1.0.0", + "query-string": "^4.1.0", + "sort-keys": "^1.0.0" + } + }, "npm-run-path": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", @@ -7376,12 +8095,12 @@ } }, "nth-check": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", - "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.0.0.tgz", + "integrity": "sha512-i4sc/Kj8htBrAiH1viZ0TgU8Y5XqCaV/FziYK6TBczxmeKm3AEFWqqF3195yKudrarqy7Zu80Ra5dobFjn9X/Q==", "dev": true, "requires": { - "boolbase": "~1.0.0" + "boolbase": "^1.0.0" } }, "number-is-nan": { @@ -7502,6 +8221,12 @@ "isobject": "^3.0.1" } }, + "omggif": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/omggif/-/omggif-1.0.10.tgz", + "integrity": "sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw==", + "dev": true + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -7554,15 +8279,6 @@ "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true }, - "os-locale": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", - "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", - "dev": true, - "requires": { - "lcid": "^1.0.0" - } - }, "os-tmpdir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", @@ -7671,12 +8387,21 @@ } }, "param-case": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/param-case/-/param-case-2.1.1.tgz", - "integrity": "sha1-35T9jPZTHs915r75oIWPvHK+Ikc=", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz", + "integrity": "sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==", "dev": true, "requires": { - "no-case": "^2.2.0" + "dot-case": "^3.0.4", + "tslib": "^2.0.3" + }, + "dependencies": { + "tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", + "dev": true + } } }, "parent-module": { @@ -7702,6 +8427,15 @@ "safe-buffer": "^5.1.1" } }, + "parse-author": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/parse-author/-/parse-author-2.0.0.tgz", + "integrity": "sha1-00YL8d3Q367tQtp1QkLmX7aEqB8=", + "dev": true, + "requires": { + "author-regex": "^1.0.0" + } + }, "parse-bmfont-ascii": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/parse-bmfont-ascii/-/parse-bmfont-ascii-1.0.6.tgz", @@ -7753,12 +8487,36 @@ "pngjs": "^3.2.0" } }, + "parse5": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz", + "integrity": "sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==", + "dev": true + }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", "dev": true }, + "pascal-case": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.2.tgz", + "integrity": "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==", + "dev": true, + "requires": { + "no-case": "^3.0.4", + "tslib": "^2.0.3" + }, + "dependencies": { + "tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", + "dev": true + } + } + }, "pascalcase": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", @@ -7845,48 +8603,11 @@ "sha.js": "^2.4.8" } }, - "pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=", - "dev": true - }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, - "phantomjs-prebuilt": { - "version": "2.1.16", - "resolved": "https://registry.npmjs.org/phantomjs-prebuilt/-/phantomjs-prebuilt-2.1.16.tgz", - "integrity": "sha1-79ISpKOWbTZHaE6ouniFSb4q7+8=", - "dev": true, - "requires": { - "es6-promise": "^4.0.3", - "extract-zip": "^1.6.5", - "fs-extra": "^1.0.0", - "hasha": "^2.2.0", - "kew": "^0.7.0", - "progress": "^1.1.8", - "request": "^2.81.0", - "request-progress": "^2.0.1", - "which": "^1.2.10" - }, - "dependencies": { - "es6-promise": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", - "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==", - "dev": true - }, - "progress": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", - "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=", - "dev": true - } - } - }, "phin": { "version": "2.9.3", "resolved": "https://registry.npmjs.org/phin/-/phin-2.9.3.tgz", @@ -7897,8 +8618,7 @@ "version": "2.2.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", - "dev": true, - "optional": true + "dev": true }, "pify": { "version": "4.0.1", @@ -7992,12 +8712,6 @@ } } }, - "pn": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", - "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==", - "dev": true - }, "pngjs": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", @@ -8094,12 +8808,39 @@ "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", "dev": true }, + "prebuild-install": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-6.1.3.tgz", + "integrity": "sha512-iqqSR84tNYQUQHRXalSKdIaM8Ov1QxOVuBNWI7+BzZWv6Ih9k75wOnH1rGQ9WWTaaLkTpxWKIciOF0KyfM74+Q==", + "dev": true, + "requires": { + "detect-libc": "^1.0.3", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^2.21.0", + "npmlog": "^4.0.1", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^3.0.3", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + } + }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", "dev": true }, + "prepend-http": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", + "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=", + "dev": true + }, "prettier": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.0.5.tgz", @@ -8116,13 +8857,21 @@ } }, "pretty-error": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/pretty-error/-/pretty-error-2.1.1.tgz", - "integrity": "sha1-X0+HyPkeWuPzuoerTPXgOxoX8aM=", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/pretty-error/-/pretty-error-2.1.2.tgz", + "integrity": "sha512-EY5oDzmsX5wvuynAByrmY0P0hcp+QpnAKbJng2A2MPjVKXCxrDSUkzghVJ4ZGPIv+JC4gX8fPUWscC0RtjsWGw==", "dev": true, "requires": { - "renderkid": "^2.0.1", - "utila": "~0.4" + "lodash": "^4.17.20", + "renderkid": "^2.0.4" + }, + "dependencies": { + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + } } }, "private": { @@ -8257,6 +9006,16 @@ "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", "dev": true }, + "query-string": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-4.3.4.tgz", + "integrity": "sha1-u7aTucqRXCMlFbIosaArYJBD2+s=", + "dev": true, + "requires": { + "object-assign": "^4.1.0", + "strict-uri-encode": "^1.0.0" + } + }, "querystring": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", @@ -8269,6 +9028,15 @@ "integrity": "sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM=", "dev": true }, + "queue": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/queue/-/queue-6.0.1.tgz", + "integrity": "sha512-AJBQabRCCNr9ANq8v77RJEv73DPbn55cdTb+Giq4X0AVnNVZvMHlYp7XlQiN+1npCZj1DuSmaA2hYVUUDgxFDg==", + "dev": true, + "requires": { + "inherits": "~2.0.3" + } + }, "raf": { "version": "3.4.1", "resolved": "https://registry.npmjs.org/raf/-/raf-3.4.1.tgz", @@ -8335,6 +9103,18 @@ } } }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + } + }, "react": { "version": "16.8.6", "resolved": "https://registry.npmjs.org/react/-/react-16.8.6.tgz", @@ -8829,20 +9609,19 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", "integrity": "sha1-wkvOKig62tW8P1jg1IJJuSN52O8=", - "dev": true, - "optional": true + "dev": true }, "renderkid": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/renderkid/-/renderkid-2.0.3.tgz", - "integrity": "sha512-z8CLQp7EZBPCwCnncgf9C4XAi3WR0dv+uWu/PjIyhhAb5d6IJ/QZqlHFprHeKT+59//V6BNUsLbvN8+2LarxGA==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/renderkid/-/renderkid-2.0.7.tgz", + "integrity": "sha512-oCcFyxaMrKsKcTY59qnCAtmDVSLfPbrv6A3tVbPdFMMrv5jaK10V6m40cKsoPNhAqN6rmHW9sswW4o3ruSrwUQ==", "dev": true, "requires": { - "css-select": "^1.1.0", - "dom-converter": "^0.2", - "htmlparser2": "^3.3.0", - "strip-ansi": "^3.0.0", - "utila": "^0.4.0" + "css-select": "^4.1.3", + "dom-converter": "^0.2.0", + "htmlparser2": "^6.1.0", + "lodash": "^4.17.21", + "strip-ansi": "^3.0.1" }, "dependencies": { "ansi-regex": { @@ -8851,34 +9630,12 @@ "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", "dev": true }, - "css-select": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", - "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", - "dev": true, - "requires": { - "boolbase": "~1.0.0", - "css-what": "2.1", - "domutils": "1.5.1", - "nth-check": "~1.0.1" - } - }, - "css-what": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", - "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==", + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true }, - "domutils": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", - "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", - "dev": true, - "requires": { - "dom-serializer": "0", - "domelementtype": "1" - } - }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", @@ -8910,9 +9667,9 @@ } }, "replace-ext": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/replace-ext/-/replace-ext-0.0.1.tgz", - "integrity": "sha1-KbvZIHinOfC8zitO5B6DeVNSKSQ=", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/replace-ext/-/replace-ext-1.0.1.tgz", + "integrity": "sha512-yD5BHCe7quCgBph4rMQ+0KkIRKwWCrHDOX1p1Gp6HwjPM5kVoCdKGNhN7ydqqsX6lJEnQDKZ/tFMiEdQ1dvPEw==", "dev": true }, "request": { @@ -8943,15 +9700,6 @@ "uuid": "^3.3.2" } }, - "request-progress": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/request-progress/-/request-progress-2.0.1.tgz", - "integrity": "sha1-XTa7V5YcZzqlt4jbyBQf3yO0Tgg=", - "dev": true, - "requires": { - "throttleit": "^1.0.0" - } - }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -8988,11 +9736,70 @@ "integrity": "sha1-WtAUcJnROp84qnuZrx1ueGZu038=", "dev": true }, + "file-type": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", + "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek=", + "dev": true + }, + "jimp": { + "version": "0.2.28", + "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.2.28.tgz", + "integrity": "sha1-3VKak3GQ9ClXp5N9Gsw6d2KZbqI=", + "dev": true, + "requires": { + "bignumber.js": "^2.1.0", + "bmp-js": "0.0.3", + "es6-promise": "^3.0.2", + "exif-parser": "^0.1.9", + "file-type": "^3.1.0", + "jpeg-js": "^0.2.0", + "load-bmfont": "^1.2.3", + "mime": "^1.3.4", + "mkdirp": "0.5.1", + "pixelmatch": "^4.0.0", + "pngjs": "^3.0.0", + "read-chunk": "^1.0.1", + "request": "^2.65.0", + "stream-to-buffer": "^0.1.0", + "tinycolor2": "^1.1.2", + "url-regex": "^3.0.0" + }, + "dependencies": { + "bmp-js": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.0.3.tgz", + "integrity": "sha1-ZBE+nHzxICs3btYHvzBibr5XsYo=", + "dev": true + }, + "jpeg-js": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.2.0.tgz", + "integrity": "sha1-U+RI7J0mPmgyZkZ+lELSxaLvVII=", + "dev": true + } + } + }, "jpeg-js": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.1.2.tgz", "integrity": "sha1-E1uZLAV1yYXPoPSUoyJ+0jhYPs4=", "dev": true + }, + "minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "dev": true, + "requires": { + "minimist": "0.0.8" + } } } }, @@ -9436,11 +10243,53 @@ } } }, - "shallowequal": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz", - "integrity": "sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ==" - }, + "shallowequal": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz", + "integrity": "sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ==" + }, + "sharp": { + "version": "0.28.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.28.3.tgz", + "integrity": "sha512-21GEP45Rmr7q2qcmdnjDkNP04Ooh5v0laGS5FDpojOO84D1DJwUijLiSq8XNNM6e8aGXYtoYRh3sVNdm8NodMA==", + "dev": true, + "requires": { + "color": "^3.1.3", + "detect-libc": "^1.0.3", + "node-addon-api": "^3.2.0", + "prebuild-install": "^6.1.2", + "semver": "^7.3.5", + "simple-get": "^3.1.0", + "tar-fs": "^2.1.1", + "tunnel-agent": "^0.6.0" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, "shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", @@ -9459,12 +10308,46 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" }, + "simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "dev": true + }, + "simple-get": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", + "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==", + "dev": true, + "requires": { + "decompress-response": "^4.2.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "simple-html-tokenizer": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/simple-html-tokenizer/-/simple-html-tokenizer-0.1.1.tgz", "integrity": "sha1-BcLuxXn//+FFoDCsJs/qYbmA+r4=", "dev": true }, + "simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", + "dev": true, + "requires": { + "is-arrayish": "^0.3.1" + }, + "dependencies": { + "is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "dev": true + } + } + }, "slice-ansi": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", @@ -9586,6 +10469,15 @@ } } }, + "sort-keys": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz", + "integrity": "sha1-RBttTTRnmPG05J6JIK37oOVD+a0=", + "dev": true, + "requires": { + "is-plain-obj": "^1.0.0" + } + }, "source-list-map": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz", @@ -9888,6 +10780,12 @@ "stream-to": "~0.2.0" } }, + "strict-uri-encode": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", + "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=", + "dev": true + }, "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", @@ -9981,40 +10879,14 @@ } }, "svg-inline-loader": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/svg-inline-loader/-/svg-inline-loader-0.8.0.tgz", - "integrity": "sha512-rynplY2eXFrdNomL1FvyTFQlP+dx0WqbzHglmNtA9M4IHRC3no2aPAl3ny9lUpJzFzFMZfWRK5YIclNU+FRePA==", + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/svg-inline-loader/-/svg-inline-loader-0.8.2.tgz", + "integrity": "sha512-kbrcEh5n5JkypaSC152eGfGcnT4lkR0eSfvefaUJkLqgGjRQJyKDvvEE/CCv5aTSdfXuc+N98w16iAojhShI3g==", "dev": true, "requires": { - "loader-utils": "^0.2.11", + "loader-utils": "^1.1.0", "object-assign": "^4.0.1", "simple-html-tokenizer": "^0.1.1" - }, - "dependencies": { - "big.js": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/big.js/-/big.js-3.2.0.tgz", - "integrity": "sha512-+hN/Zh2D08Mx65pZ/4g5bsmNiZUuChDiQfTUQ7qJr4/kuopCr88xZsAXv6mBoZEsUI4OuGHlX59qE94K2mMW8Q==", - "dev": true - }, - "json5": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-0.5.1.tgz", - "integrity": "sha1-Hq3nrMASA0rYTiOWdn6tn6VJWCE=", - "dev": true - }, - "loader-utils": { - "version": "0.2.17", - "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-0.2.17.tgz", - "integrity": "sha1-+G5jdNQyBabmxg6RlvF8Apm/s0g=", - "dev": true, - "requires": { - "big.js": "^3.1.3", - "emojis-list": "^2.0.0", - "json5": "^0.5.0", - "object-assign": "^4.0.1" - } - } } }, "svg-react-loader": { @@ -10133,17 +11005,6 @@ "svg.js": "^2.6.5" } }, - "svg2png": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/svg2png/-/svg2png-3.0.1.tgz", - "integrity": "sha1-omRNaLAjGsAK9DGqFjcU/xcQZEc=", - "dev": true, - "requires": { - "phantomjs-prebuilt": "^2.1.10", - "pn": "^1.0.0", - "yargs": "^3.31.0" - } - }, "symbol-observable": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", @@ -10205,6 +11066,59 @@ "inherits": "2" } }, + "tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dev": true, + "requires": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, + "requires": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "requires": { + "safe-buffer": "~5.2.0" + } + } + } + }, "terser": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/terser/-/terser-4.4.2.tgz", @@ -10261,12 +11175,6 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, - "throttleit": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-1.0.0.tgz", - "integrity": "sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw=", - "dev": true - }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -10324,6 +11232,12 @@ "setimmediate": "^1.0.4" } }, + "timm": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/timm/-/timm-1.7.1.tgz", + "integrity": "sha512-IjZc9KIotudix8bMaBW6QvMuq64BrJWFs1+4V0lXwWGQZwH+LnX87doAYhem4caOEusRP9/g6jVDQmZ8XOk1nw==", + "dev": true + }, "tiny-invariant": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.0.6.tgz", @@ -10335,9 +11249,9 @@ "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==" }, "tinycolor2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.4.1.tgz", - "integrity": "sha1-9PrTM0R7wLB9TcjpIJ2POaisd+g=", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.4.2.tgz", + "integrity": "sha512-vJhccZPs965sV/L2sU4oRQVAos0pQXwsvTLkWYdqJ+a8Q5kPFzJTuOFwy7UniPli44NKQGAglksjvOcpo95aZA==", "dev": true }, "tmp": { @@ -10420,12 +11334,6 @@ "repeat-string": "^1.6.1" } }, - "toposort": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/toposort/-/toposort-1.0.7.tgz", - "integrity": "sha1-LmhELZ9k7HILjMieZEOsbKqVACk=", - "dev": true - }, "tough-cookie": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", @@ -10521,36 +11429,6 @@ "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.20.tgz", "integrity": "sha512-8OaIKfzL5cpx8eCMAhhvTlft8GYF8b2eQr6JkCyVdrgjcytyOmPCXrqXFcUnhonRpLlh5yxEZVohm6mzaowUOw==" }, - "uglify-js": { - "version": "3.4.10", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.4.10.tgz", - "integrity": "sha512-Y2VsbPVs0FIshJztycsO2SfPk7/KAF/T72qzv9u5EpQ4kB2hQoHlhNQTsNyy6ul7lQtqJN/AoWeS23OzEiEFxw==", - "dev": true, - "requires": { - "commander": "~2.19.0", - "source-map": "~0.6.1" - }, - "dependencies": { - "commander": { - "version": "2.19.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.19.0.tgz", - "integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==", - "dev": true - }, - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true - } - } - }, - "underscore": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.9.1.tgz", - "integrity": "sha512-5/4etnCkd9c8gwgowi5/om/mYO5ajCaOgdzj/oW+0eQV9WxKBDZw5+ycmKmeaTXjInS/W0BzpGLo2xR2aBwZdg==", - "dev": true - }, "unicode-canonical-property-names-ecmascript": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz", @@ -10668,12 +11546,6 @@ "dev": true, "optional": true }, - "upper-case": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/upper-case/-/upper-case-1.1.3.tgz", - "integrity": "sha1-9rRQHC7EzdJrp4vnIilh3ndiFZg=", - "dev": true - }, "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", @@ -10744,6 +11616,15 @@ "resolved": "https://registry.npmjs.org/use-memo-one/-/use-memo-one-1.1.1.tgz", "integrity": "sha512-oFfsyun+bP7RX8X2AskHNTxu+R3QdE/RC5IefMbqptmACAA/gfol1KDD5KRzPsGMa62sWxGZw+Ui43u6x4ddoQ==" }, + "utif": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/utif/-/utif-2.0.1.tgz", + "integrity": "sha512-Z/S1fNKCicQTf375lIP9G8Sa1H/phcysstNrrSdZKj1f9g58J4NMgb5IgiEZN9/nLMPDwF0W7hdOe9Qq2IYoLg==", + "dev": true, + "requires": { + "pako": "^1.0.5" + } + }, "util": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz", @@ -10832,14 +11713,17 @@ } }, "vinyl": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/vinyl/-/vinyl-1.2.0.tgz", - "integrity": "sha1-XIgDbPVl5d8FVYv8kR+GVt8hiIQ=", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/vinyl/-/vinyl-2.2.1.tgz", + "integrity": "sha512-LII3bXRFBZLlezoG5FfZVcXflZgWP/4dCwKtxd5ky9+LOtM4CS3bIRQsmR1KMnMW07jpE8fqR2lcxPZ+8sJIcw==", "dev": true, "requires": { - "clone": "^1.0.0", - "clone-stats": "^0.0.1", - "replace-ext": "0.0.1" + "clone": "^2.1.1", + "clone-buffer": "^1.0.0", + "clone-stats": "^1.0.0", + "cloneable-readable": "^1.0.0", + "remove-trailing-separator": "^1.0.1", + "replace-ext": "^1.0.0" } }, "vm-browserify": { @@ -11256,12 +12140,6 @@ "string-width": "^1.0.2 || 2" } }, - "window-size": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.4.tgz", - "integrity": "sha1-+OGqHuWlPsW/FR/6CXQqatdpeHY=", - "dev": true - }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -11277,53 +12155,6 @@ "errno": "~0.1.7" } }, - "wrap-ansi": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", - "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", - "dev": true, - "requires": { - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "dev": true, - "requires": { - "number-is-nan": "^1.0.0" - } - }, - "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "dev": true, - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "dev": true, - "requires": { - "ansi-regex": "^2.0.0" - } - } - } - }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -11339,15 +12170,33 @@ } }, "xhr": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/xhr/-/xhr-2.5.0.tgz", - "integrity": "sha512-4nlO/14t3BNUZRXIXfXe+3N6w3s1KoxcJUUURctd64BLRe67E4gRwp4PjywtDY72fXpZ1y6Ch0VZQRY/gMPzzQ==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/xhr/-/xhr-2.6.0.tgz", + "integrity": "sha512-/eCGLb5rxjx5e3mF1A7s+pLlR6CGyqWN91fv1JgER5mVWg1MZmlhBvy9kjcsOdRk8RrIujotWyJamfyrp+WIcA==", "dev": true, "requires": { - "global": "~4.3.0", + "global": "~4.4.0", "is-function": "^1.0.1", "parse-headers": "^2.0.0", "xtend": "^4.0.0" + }, + "dependencies": { + "global": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz", + "integrity": "sha512-wv/LAoHdRE3BeTGz53FAamhGlPLhlssK45usmGFThIi4XqnBmjKQ16u+RNbP7WvigRZDxUsM0J3gcQ5yicaL0w==", + "dev": true, + "requires": { + "min-document": "^2.19.0", + "process": "^0.11.10" + } + }, + "process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=", + "dev": true + } } }, "xml-parse-from-string": { @@ -11357,13 +12206,12 @@ "dev": true }, "xml2js": { - "version": "0.4.22", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.22.tgz", - "integrity": "sha512-MWTbxAQqclRSTnehWWe5nMKzI3VmJ8ltiJEco8akcC6j3miOhjjfzKum5sId+CWhfxdOs/1xauYr8/ZDBtQiRw==", + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", "dev": true, "requires": { "sax": ">=0.6.0", - "util.promisify": "~1.0.0", "xmlbuilder": "~11.0.0" } }, @@ -11379,76 +12227,12 @@ "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", "dev": true }, - "y18n": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", - "integrity": "sha1-bRX7qITAhnnA136I53WegR4H+kE=", - "dev": true - }, "yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", "dev": true }, - "yargs": { - "version": "3.32.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.32.0.tgz", - "integrity": "sha1-AwiOnr+edWtpdRYR0qXvWRSCyZU=", - "dev": true, - "requires": { - "camelcase": "^2.0.1", - "cliui": "^3.0.3", - "decamelize": "^1.1.1", - "os-locale": "^1.4.0", - "string-width": "^1.0.1", - "window-size": "^0.1.4", - "y18n": "^3.2.0" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "dev": true - }, - "camelcase": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz", - "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8=", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "dev": true, - "requires": { - "number-is-nan": "^1.0.0" - } - }, - "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "dev": true, - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "dev": true, - "requires": { - "ansi-regex": "^2.0.0" - } - } - } - }, "yargs-parser": { "version": "13.1.2", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", @@ -11457,15 +12241,6 @@ "camelcase": "^5.0.0", "decamelize": "^1.2.0" } - }, - "yauzl": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz", - "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=", - "dev": true, - "requires": { - "fd-slicer": "~1.0.1" - } } } } diff --git a/ui/package.json b/ui/package.json index 563e5236..f8986110 100755 --- a/ui/package.json +++ b/ui/package.json @@ -64,4 +64,4 @@ "react-router-dom": "5.0.0", "svg-react-loader": "0.4.6" } -} \ No newline at end of file +} diff --git a/ui/src/components/Dashboard/CSVDownloadButton.js b/ui/src/components/Dashboard/CSVDownloadButton.js new file mode 100644 index 00000000..03cbc075 --- /dev/null +++ b/ui/src/components/Dashboard/CSVDownloadButton.js @@ -0,0 +1,37 @@ +import React from 'react'; +import {CSVLink} from "react-csv" + +const CSVDownloadButton = () => { + const data = [ + { firstName: "Warren", lastName: "Morrow", email: "sokyt@mailinator.com", age: "36" }, + { firstName: "Gwendolyn", lastName: "Galloway", email: "weciz@mailinator.com", age: "76" }, + { firstName: "Astra", lastName: "Wyatt", email: "quvyn@mailinator.com", age: "57" }, + { firstName: "Jasmine", lastName: "Wong", email: "toxazoc@mailinator.com", age: "42" }, + { firstName: "Brooke", lastName: "Mcconnell", email: "vyry@mailinator.com", age: "56" }, + { firstName: "Christen", lastName: "Haney", email: "pagevolal@mailinator.com", age: "23" }, + { firstName: "Tate", lastName: "Vega", email: "dycubo@mailinator.com", age: "87" }, + { firstName: "Amber", lastName: "Brady", email: "vyconixy@mailinator.com", age: "78" }, + { firstName: "Philip", lastName: "Whitfield", email: "velyfi@mailinator.com", age: "22" }, + { firstName: "Kitra", lastName: "Hammond", email: "fiwiloqu@mailinator.com", age: "35" }, + { firstName: "Charity", lastName: "Mathews", email: "fubigonero@mailinator.com", age: "63" } + ]; + const headers = [ + { label: "First Name", key: "firstName" }, + { label: "Last Name", key: "lastName" }, + { label: "Email", key: "email" }, + { label: "Age", key: "age" } + ]; + + const csvReport = { + data: data, + headers: headers, + filename: 'Clue_Mediator_Report.csv' + }; + return ( +
+ Export to CSV +
+ ); +}; + +export default CSVDownloadButton; \ No newline at end of file diff --git a/ui/src/components/Dashboard/Index.js b/ui/src/components/Dashboard/Index.js index cb963357..af22a688 100644 --- a/ui/src/components/Dashboard/Index.js +++ b/ui/src/components/Dashboard/Index.js @@ -7,6 +7,7 @@ import PropTypes from "prop-types"; import AccountsList from "./AccountsList"; import FilterBar from "./FilterBar"; import StatisticsBar from "./StatisticsBar"; +import CSVDownloadButton from "./CSVDownloadButton"; import ResourceScanning from "./ResourceScanning"; import ResourcesChart from "./ResourcesChart"; import ResourcesCharts from "./ResourcesCharts"; @@ -64,6 +65,7 @@ const DashboardIndex = ({ + From b95df0232c19897bd593cc8be40ea5360ede7642 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Tue, 22 Jun 2021 15:56:38 +0200 Subject: [PATCH 25/68] Fix ESLint Errors --- ui/package-lock.json | 6 + ui/package.json | 1 + .../components/Dashboard/CSVDownloadButton.js | 119 +++++++++++++----- 3 files changed, 94 insertions(+), 32 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index 4b863431..2f57e4e3 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -9163,6 +9163,12 @@ } } }, + "react-csv": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/react-csv/-/react-csv-2.0.3.tgz", + "integrity": "sha512-exyAdFLAxtuM4wNwLYrlKyPYLiJ7e0mv9tqPAd3kq+k1CiJFtznevR3yP0icv5q/y200w+lzNgi7TQn1Wrhu0w==", + "dev": true + }, "react-display-name": { "version": "0.2.5", "resolved": "https://registry.npmjs.org/react-display-name/-/react-display-name-0.2.5.tgz", diff --git a/ui/package.json b/ui/package.json index f8986110..933b18e3 100755 --- a/ui/package.json +++ b/ui/package.json @@ -35,6 +35,7 @@ "node-sass": "4.14.1", "prettier": "2.0.5", "react": "16.8.6", + "react-csv": "^2.0.3", "react-dom": "16.8.6", "react-hot-loader": "4.8.4", "react-inlinesvg": "1.1.7", diff --git a/ui/src/components/Dashboard/CSVDownloadButton.js b/ui/src/components/Dashboard/CSVDownloadButton.js index 03cbc075..08e983ad 100644 --- a/ui/src/components/Dashboard/CSVDownloadButton.js +++ b/ui/src/components/Dashboard/CSVDownloadButton.js @@ -1,37 +1,92 @@ -import React from 'react'; -import {CSVLink} from "react-csv" +import React from "react"; +import { CSVLink } from "react-csv"; const CSVDownloadButton = () => { - const data = [ - { firstName: "Warren", lastName: "Morrow", email: "sokyt@mailinator.com", age: "36" }, - { firstName: "Gwendolyn", lastName: "Galloway", email: "weciz@mailinator.com", age: "76" }, - { firstName: "Astra", lastName: "Wyatt", email: "quvyn@mailinator.com", age: "57" }, - { firstName: "Jasmine", lastName: "Wong", email: "toxazoc@mailinator.com", age: "42" }, - { firstName: "Brooke", lastName: "Mcconnell", email: "vyry@mailinator.com", age: "56" }, - { firstName: "Christen", lastName: "Haney", email: "pagevolal@mailinator.com", age: "23" }, - { firstName: "Tate", lastName: "Vega", email: "dycubo@mailinator.com", age: "87" }, - { firstName: "Amber", lastName: "Brady", email: "vyconixy@mailinator.com", age: "78" }, - { firstName: "Philip", lastName: "Whitfield", email: "velyfi@mailinator.com", age: "22" }, - { firstName: "Kitra", lastName: "Hammond", email: "fiwiloqu@mailinator.com", age: "35" }, - { firstName: "Charity", lastName: "Mathews", email: "fubigonero@mailinator.com", age: "63" } - ]; - const headers = [ - { label: "First Name", key: "firstName" }, - { label: "Last Name", key: "lastName" }, - { label: "Email", key: "email" }, - { label: "Age", key: "age" } - ]; + const data = [ + { + firstName: "Warren", + lastName: "Morrow", + email: "sokyt@mailinator.com", + age: "36", + }, + { + firstName: "Gwendolyn", + lastName: "Galloway", + email: "weciz@mailinator.com", + age: "76", + }, + { + firstName: "Astra", + lastName: "Wyatt", + email: "quvyn@mailinator.com", + age: "57", + }, + { + firstName: "Jasmine", + lastName: "Wong", + email: "toxazoc@mailinator.com", + age: "42", + }, + { + firstName: "Brooke", + lastName: "Mcconnell", + email: "vyry@mailinator.com", + age: "56", + }, + { + firstName: "Christen", + lastName: "Haney", + email: "pagevolal@mailinator.com", + age: "23", + }, + { + firstName: "Tate", + lastName: "Vega", + email: "dycubo@mailinator.com", + age: "87", + }, + { + firstName: "Amber", + lastName: "Brady", + email: "vyconixy@mailinator.com", + age: "78", + }, + { + firstName: "Philip", + lastName: "Whitfield", + email: "velyfi@mailinator.com", + age: "22", + }, + { + firstName: "Kitra", + lastName: "Hammond", + email: "fiwiloqu@mailinator.com", + age: "35", + }, + { + firstName: "Charity", + lastName: "Mathews", + email: "fubigonero@mailinator.com", + age: "63", + }, + ]; + const headers = [ + { label: "First Name", key: "firstName" }, + { label: "Last Name", key: "lastName" }, + { label: "Email", key: "email" }, + { label: "Age", key: "age" }, + ]; - const csvReport = { - data: data, - headers: headers, - filename: 'Clue_Mediator_Report.csv' - }; - return ( -
- Export to CSV -
- ); + const csvReport = { + data: data, + headers: headers, + filename: "Clue_Mediator_Report.csv", + }; + return ( +
+ Export to CSV +
+ ); }; -export default CSVDownloadButton; \ No newline at end of file +export default CSVDownloadButton; From 9429c6f05560b0e32d516d0c2993ccc4b262b5e0 Mon Sep 17 00:00:00 2001 From: DerOtt Date: Fri, 25 Jun 2021 14:13:50 +0200 Subject: [PATCH 26/68] I dont care --- api/route.go | 78 +++++++++ api/server.go | 1 + go.sum | 6 - .../components/Dashboard/CSVDownloadButton.js | 151 ++++++++++++++---- ui/src/components/Dashboard/Index.js | 2 - ui/src/components/Dashboard/ResourcesChart.js | 2 + ui/src/services/report.service.js | 18 +++ ui/src/services/resources.service.js | 16 ++ 8 files changed, 231 insertions(+), 43 deletions(-) create mode 100644 ui/src/services/report.service.js diff --git a/api/route.go b/api/route.go index 756c2f32..106e39bb 100644 --- a/api/route.go +++ b/api/route.go @@ -4,6 +4,7 @@ import ( "encoding/json" "finala/api/httpparameters" "finala/api/storage" + "fmt" "io/ioutil" "net/http" "net/url" @@ -200,3 +201,80 @@ func (server *Server) VersionHandler(resp http.ResponseWriter, req *http.Request } server.JSONWrite(resp, http.StatusOK, version) } + +//Returns json thingy wingy dingy i dont know how +func (server *Server) GetReport(resp http.ResponseWriter, req *http.Request) { + queryParams := req.URL.Query() + params := mux.Vars(req) + executionID := params["executionID"] + filters := httpparameters.GetFilterQueryParamWithOutPrefix(queryParamFilterPrefix, queryParams) + + response, err := server.storage.GetSummary(executionID, filters) + if err != nil { + server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) + return + + } + + var result []map[string]interface{} + var attributeList []string + + for resourceName, resourceSummary := range response { + fmt.Println(resourceName, resourceSummary) //sanity test + if resourceSummary.ResourceCount > 0 { + resourcesList, err := server.storage.GetResources(resourceName, executionID, filters) + + if err != nil { + server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) + continue + } + + data, ok := resourcesList[0]["Data"].(map[string]interface{}) + if !ok { + //screw your log + continue + } + for key := range data { + exists := false + for index := range attributeList { + if attributeList[index] == key { + exists = true + } + } + if !exists { + attributeList = append(attributeList, key) + } + } + + } + } + + for resourceName, resourceSummary := range response { + fmt.Println(resourceName, resourceSummary) //sanity test + if resourceSummary.ResourceCount > 0 { + resourcesList, err := server.storage.GetResources(resourceName, executionID, filters) + if err != nil { + server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) + continue + } + for _, element := range resourcesList { + data, ok := element["Data"].(map[string]interface{}) + if !ok { + //screw your log + continue + } + + for _, attrName := range attributeList { + _, ok := data[attrName] + if !ok { + data[attrName] = "" + } + } + result = append(result, data) + } + + } + } + + server.JSONWrite(resp, http.StatusOK, result) +} diff --git a/api/server.go b/api/server.go index 357c9c69..aed7b6b5 100644 --- a/api/server.go +++ b/api/server.go @@ -88,6 +88,7 @@ func (server *Server) BindEndpoints() { server.router.HandleFunc("/api/v1/detect-events/{executionID}", server.DetectEvents).Methods("POST") server.router.HandleFunc("/api/v1/version", server.VersionHandler).Methods("GET") server.router.HandleFunc("/api/v1/health", server.HealthCheckHandler).Methods("GET") + server.router.HandleFunc("/api/v1/getReport/{executionID}", server.GetReport).Methods("GET") server.router.NotFoundHandler = http.HandlerFunc(server.NotFoundRoute) } diff --git a/go.sum b/go.sum index 49880209..d4555057 100644 --- a/go.sum +++ b/go.sum @@ -71,7 +71,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -125,7 +124,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -141,7 +139,6 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -172,9 +169,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd h1:r7DufRZuZbWB7j439YfAzP8RPDa9unLkpwQKUYbIMPI= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f h1:68K/z8GLUxV76xGSqwTWw2gyk/jwn79LUL43rES2g8o= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -198,7 +193,6 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/ui/src/components/Dashboard/CSVDownloadButton.js b/ui/src/components/Dashboard/CSVDownloadButton.js index 03cbc075..13b6cefe 100644 --- a/ui/src/components/Dashboard/CSVDownloadButton.js +++ b/ui/src/components/Dashboard/CSVDownloadButton.js @@ -1,37 +1,118 @@ -import React from 'react'; -import {CSVLink} from "react-csv" - -const CSVDownloadButton = () => { - const data = [ - { firstName: "Warren", lastName: "Morrow", email: "sokyt@mailinator.com", age: "36" }, - { firstName: "Gwendolyn", lastName: "Galloway", email: "weciz@mailinator.com", age: "76" }, - { firstName: "Astra", lastName: "Wyatt", email: "quvyn@mailinator.com", age: "57" }, - { firstName: "Jasmine", lastName: "Wong", email: "toxazoc@mailinator.com", age: "42" }, - { firstName: "Brooke", lastName: "Mcconnell", email: "vyry@mailinator.com", age: "56" }, - { firstName: "Christen", lastName: "Haney", email: "pagevolal@mailinator.com", age: "23" }, - { firstName: "Tate", lastName: "Vega", email: "dycubo@mailinator.com", age: "87" }, - { firstName: "Amber", lastName: "Brady", email: "vyconixy@mailinator.com", age: "78" }, - { firstName: "Philip", lastName: "Whitfield", email: "velyfi@mailinator.com", age: "22" }, - { firstName: "Kitra", lastName: "Hammond", email: "fiwiloqu@mailinator.com", age: "35" }, - { firstName: "Charity", lastName: "Mathews", email: "fubigonero@mailinator.com", age: "63" } - ]; - const headers = [ - { label: "First Name", key: "firstName" }, - { label: "Last Name", key: "lastName" }, - { label: "Email", key: "email" }, - { label: "Age", key: "age" } - ]; - - const csvReport = { - data: data, - headers: headers, - filename: 'Clue_Mediator_Report.csv' - }; - return ( -
- Export to CSV -
- ); +import React from "react"; +import { CSVLink } from "react-csv"; +import { ResourcesService } from "services/resources.service"; +import { connect } from "react-redux"; +import { PropTypes } from "@material-ui/core"; + +const CSVDownloadButton = ({ executionId, filters }) => { + + //Testdata to Be ignored + const data = [ + { + firstName: "Warren", + lastName: "Morrow", + email: "sokyt@mailinator.com", + age: "36", + }, + { + firstName: "Gwendolyn", + lastName: "Galloway", + email: "weciz@mailinator.com", + age: "76", + }, + { + firstName: "Astra", + lastName: "Wyatt", + email: "quvyn@mailinator.com", + age: "57", + }, + { + firstName: "Jasmine", + lastName: "Wong", + email: "toxazoc@mailinator.com", + age: "42", + }, + { + firstName: "Brooke", + lastName: "Mcconnell", + email: "vyry@mailinator.com", + age: "56", + }, + { + firstName: "Christen", + lastName: "Haney", + email: "pagevolal@mailinator.com", + age: "23", + }, + { + firstName: "Tate", + lastName: "Vega", + email: "dycubo@mailinator.com", + age: "87", + }, + { + firstName: "Amber", + lastName: "Brady", + email: "vyconixy@mailinator.com", + age: "78", + }, + { + firstName: "Philip", + lastName: "Whitfield", + email: "velyfi@mailinator.com", + age: "22", + }, + { + firstName: "Kitra", + lastName: "Hammond", + email: "fiwiloqu@mailinator.com", + age: "35", + }, + { + firstName: "Charity", + lastName: "Mathews", + email: "fubigonero@mailinator.com", + age: "63", + }, + ]; + const headers = [ + { label: "First Name", key: "firstName" }, + { label: "Last Name", key: "lastName" }, + { label: "Email", key: "email" }, + { label: "Age", key: "age" }, + ]; + + const csvReport = { + data: data, + headers: headers, + filename: "Clue_Mediator_Report.csv", + }; + + //Testdata to Be ignored + + const downloadReport = async () => { + data = await ResourcesService.GetReport(executionId, filters); //tobefixed + + }; + return ( +
+ + +
+ ); }; -export default CSVDownloadButton; \ No newline at end of file + //Todo + // -> Get the Execution ID and pass it do downloadReport + +CSVDownloadButton.propTypes = { + executionId: PropTypes.string, + filters: PropTypes.array, +}; + +const mapStateToProps = (state) => ({ + executionId: state.executions.current, + filters: state.filters.filters, +}); + +export default connect(mapStateToProps)(CSVDownloadButton); diff --git a/ui/src/components/Dashboard/Index.js b/ui/src/components/Dashboard/Index.js index af22a688..cb963357 100644 --- a/ui/src/components/Dashboard/Index.js +++ b/ui/src/components/Dashboard/Index.js @@ -7,7 +7,6 @@ import PropTypes from "prop-types"; import AccountsList from "./AccountsList"; import FilterBar from "./FilterBar"; import StatisticsBar from "./StatisticsBar"; -import CSVDownloadButton from "./CSVDownloadButton"; import ResourceScanning from "./ResourceScanning"; import ResourcesChart from "./ResourcesChart"; import ResourcesCharts from "./ResourcesCharts"; @@ -65,7 +64,6 @@ const DashboardIndex = ({ - diff --git a/ui/src/components/Dashboard/ResourcesChart.js b/ui/src/components/Dashboard/ResourcesChart.js index fa1fa8ce..77d60d8e 100644 --- a/ui/src/components/Dashboard/ResourcesChart.js +++ b/ui/src/components/Dashboard/ResourcesChart.js @@ -6,6 +6,7 @@ import Chart from "react-apexcharts"; import { titleDirective } from "../../utils/Title"; import { MoneyDirective } from "../../utils/Money"; import { setHistory } from "../../utils/History"; +import CSVDownloadButton from "./CSVDownloadButton"; import { Box, @@ -186,6 +187,7 @@ const ResourcesChart = ({ {!isResourceListLoading && sortedResources.length > 0 && ( +

{account ? `${accounts[account].Name} (${accounts[account].ID}):` diff --git a/ui/src/services/report.service.js b/ui/src/services/report.service.js new file mode 100644 index 00000000..f75b8630 --- /dev/null +++ b/ui/src/services/report.service.js @@ -0,0 +1,18 @@ +import { http } from "./request.service"; + +export const AccsService = { + fetchSummary, +}; + +/** + * + * @param {string} executionId execution to query + */ +function fetchSummary(executionId) { + return http + .send(`api/v1/getReport/${executionId}`, `get`) + .then(this.handleResponse) + .then((response) => { + return response; + }); +} diff --git a/ui/src/services/resources.service.js b/ui/src/services/resources.service.js index fa4604a6..98a9aba6 100644 --- a/ui/src/services/resources.service.js +++ b/ui/src/services/resources.service.js @@ -4,6 +4,7 @@ export const ResourcesService = { GetExecutions, Summary, GetContent, + GetReport, }; /** @@ -89,3 +90,18 @@ function GetContent(name, executionID, filters = []) { return response; }); } + +function GetReport(executionId, filters) { + const params = { + ...getTransformedFilters(filters), + }; + const searchParams = decodeURIComponent( + new window.URLSearchParams(params).toString() + ); + return http + .send(`api/v1/getReport/${executionId}?${searchParams}`, `get`) + .then(this.handleResponse) + .then((response) => { + return response; + }); +} From bcc2b72c37b8f0645a4beac884a0bccd94954374 Mon Sep 17 00:00:00 2001 From: DerOtt Date: Fri, 25 Jun 2021 14:24:00 +0200 Subject: [PATCH 27/68] I cant care less --- .../components/Dashboard/CSVDownloadButton.js | 90 ++++--------------- 1 file changed, 15 insertions(+), 75 deletions(-) diff --git a/ui/src/components/Dashboard/CSVDownloadButton.js b/ui/src/components/Dashboard/CSVDownloadButton.js index 13b6cefe..e1b3f9da 100644 --- a/ui/src/components/Dashboard/CSVDownloadButton.js +++ b/ui/src/components/Dashboard/CSVDownloadButton.js @@ -3,78 +3,11 @@ import { CSVLink } from "react-csv"; import { ResourcesService } from "services/resources.service"; import { connect } from "react-redux"; import { PropTypes } from "@material-ui/core"; +import { useState } from "react"; const CSVDownloadButton = ({ executionId, filters }) => { - + const [data, setData] = useState([]); //Testdata to Be ignored - const data = [ - { - firstName: "Warren", - lastName: "Morrow", - email: "sokyt@mailinator.com", - age: "36", - }, - { - firstName: "Gwendolyn", - lastName: "Galloway", - email: "weciz@mailinator.com", - age: "76", - }, - { - firstName: "Astra", - lastName: "Wyatt", - email: "quvyn@mailinator.com", - age: "57", - }, - { - firstName: "Jasmine", - lastName: "Wong", - email: "toxazoc@mailinator.com", - age: "42", - }, - { - firstName: "Brooke", - lastName: "Mcconnell", - email: "vyry@mailinator.com", - age: "56", - }, - { - firstName: "Christen", - lastName: "Haney", - email: "pagevolal@mailinator.com", - age: "23", - }, - { - firstName: "Tate", - lastName: "Vega", - email: "dycubo@mailinator.com", - age: "87", - }, - { - firstName: "Amber", - lastName: "Brady", - email: "vyconixy@mailinator.com", - age: "78", - }, - { - firstName: "Philip", - lastName: "Whitfield", - email: "velyfi@mailinator.com", - age: "22", - }, - { - firstName: "Kitra", - lastName: "Hammond", - email: "fiwiloqu@mailinator.com", - age: "35", - }, - { - firstName: "Charity", - lastName: "Mathews", - email: "fubigonero@mailinator.com", - age: "63", - }, - ]; const headers = [ { label: "First Name", key: "firstName" }, { label: "Last Name", key: "lastName" }, @@ -91,19 +24,26 @@ const CSVDownloadButton = ({ executionId, filters }) => { //Testdata to Be ignored const downloadReport = async () => { - data = await ResourcesService.GetReport(executionId, filters); //tobefixed - + setData(await ResourcesService.GetReport(executionId, filters)); //tobefixed }; return (
- - + +
); }; - //Todo - // -> Get the Execution ID and pass it do downloadReport +//Todo +// -> Get the Execution ID and pass it do downloadReport CSVDownloadButton.propTypes = { executionId: PropTypes.string, From 47e3a84067a72b37f8cefa40a065322c7cde81a2 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Fri, 25 Jun 2021 17:50:22 +0200 Subject: [PATCH 28/68] Fix csvDownload functionality --- api/route.go | 24 ++++++++++- api/server.go | 2 +- ui/package-lock.json | 31 +++++++++++++- ui/package.json | 2 + .../components/Dashboard/CSVDownloadButton.js | 42 ++++++++----------- ui/src/components/Dashboard/ResourcesChart.js | 2 +- ui/src/services/report.service.js | 18 -------- ui/src/services/resources.service.js | 5 +-- 8 files changed, 75 insertions(+), 51 deletions(-) delete mode 100644 ui/src/services/report.service.js diff --git a/api/route.go b/api/route.go index 106e39bb..4dd929c3 100644 --- a/api/route.go +++ b/api/route.go @@ -209,7 +209,16 @@ func (server *Server) GetReport(resp http.ResponseWriter, req *http.Request) { executionID := params["executionID"] filters := httpparameters.GetFilterQueryParamWithOutPrefix(queryParamFilterPrefix, queryParams) - response, err := server.storage.GetSummary(executionID, filters) + log.WithFields(log.Fields{ + "filter": filters, + }).Info("filter") + + filterForSummary := make(map[string]string) + for filterKey, filterValue := range filters { + filterForSummary[filterKey] = filterValue + } + + response, err := server.storage.GetSummary(executionID, filterForSummary) if err != nil { server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) return @@ -229,12 +238,21 @@ func (server *Server) GetReport(resp http.ResponseWriter, req *http.Request) { continue } + log.WithFields(log.Fields{ + "name": resourceName, + "executionID": executionID, + "filter": filters, + }).Info(resourcesList) + data, ok := resourcesList[0]["Data"].(map[string]interface{}) if !ok { //screw your log continue } for key := range data { + if key == "Tag" { + continue + } exists := false for index := range attributeList { if attributeList[index] == key { @@ -264,10 +282,12 @@ func (server *Server) GetReport(resp http.ResponseWriter, req *http.Request) { continue } + delete(data, "Tag") + for _, attrName := range attributeList { _, ok := data[attrName] if !ok { - data[attrName] = "" + data[attrName] = nil } } result = append(result, data) diff --git a/api/server.go b/api/server.go index aed7b6b5..eea5c906 100644 --- a/api/server.go +++ b/api/server.go @@ -88,7 +88,7 @@ func (server *Server) BindEndpoints() { server.router.HandleFunc("/api/v1/detect-events/{executionID}", server.DetectEvents).Methods("POST") server.router.HandleFunc("/api/v1/version", server.VersionHandler).Methods("GET") server.router.HandleFunc("/api/v1/health", server.HealthCheckHandler).Methods("GET") - server.router.HandleFunc("/api/v1/getReport/{executionID}", server.GetReport).Methods("GET") + server.router.HandleFunc("/api/v1/report/{executionID}", server.GetReport).Methods("GET") server.router.NotFoundHandler = http.HandlerFunc(server.NotFoundRoute) } diff --git a/ui/package-lock.json b/ui/package-lock.json index 2f57e4e3..b3083a31 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -4164,6 +4164,11 @@ "util-deprecate": "~1.0.1" } }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + }, "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -10183,6 +10188,15 @@ "send": "0.16.2" } }, + "services": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/services/-/services-0.0.3.tgz", + "integrity": "sha1-M338romhcaEfv8FyG2liFmvsB2M=", + "requires": { + "underscore": ">=1.3.0", + "underscore.string": ">=2.0.0" + } + }, "set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", @@ -10573,8 +10587,7 @@ "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", - "dev": true + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" }, "sshpk": { "version": "1.16.1", @@ -11435,6 +11448,20 @@ "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.20.tgz", "integrity": "sha512-8OaIKfzL5cpx8eCMAhhvTlft8GYF8b2eQr6JkCyVdrgjcytyOmPCXrqXFcUnhonRpLlh5yxEZVohm6mzaowUOw==" }, + "underscore": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.1.tgz", + "integrity": "sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g==" + }, + "underscore.string": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-3.3.5.tgz", + "integrity": "sha512-g+dpmgn+XBneLmXXo+sGlW5xQEt4ErkS3mgeN2GFbremYeMBSJKr9Wf2KJplQVaiPY/f7FN6atosWYNm9ovrYg==", + "requires": { + "sprintf-js": "^1.0.3", + "util-deprecate": "^1.0.2" + } + }, "unicode-canonical-property-names-ecmascript": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz", diff --git a/ui/package.json b/ui/package.json index 933b18e3..a9caae87 100755 --- a/ui/package.json +++ b/ui/package.json @@ -34,6 +34,7 @@ "mini-css-extract-plugin": "0.10.0", "node-sass": "4.14.1", "prettier": "2.0.5", + "prop-types": "^15.7.2", "react": "16.8.6", "react-csv": "^2.0.3", "react-dom": "16.8.6", @@ -63,6 +64,7 @@ "react-apexcharts": "1.3.7", "react-history": "0.18.2", "react-router-dom": "5.0.0", + "services": "0.0.3", "svg-react-loader": "0.4.6" } } diff --git a/ui/src/components/Dashboard/CSVDownloadButton.js b/ui/src/components/Dashboard/CSVDownloadButton.js index e1b3f9da..e7f52e9d 100644 --- a/ui/src/components/Dashboard/CSVDownloadButton.js +++ b/ui/src/components/Dashboard/CSVDownloadButton.js @@ -1,31 +1,27 @@ -import React from "react"; +import React, { useState } from "react"; import { CSVLink } from "react-csv"; import { ResourcesService } from "services/resources.service"; import { connect } from "react-redux"; -import { PropTypes } from "@material-ui/core"; -import { useState } from "react"; +import PropTypes from "prop-types"; -const CSVDownloadButton = ({ executionId, filters }) => { +const CSVDownloadButton = ({ currentExecution, filters }) => { const [data, setData] = useState([]); - //Testdata to Be ignored - const headers = [ - { label: "First Name", key: "firstName" }, - { label: "Last Name", key: "lastName" }, - { label: "Email", key: "email" }, - { label: "Age", key: "age" }, - ]; - - const csvReport = { - data: data, - headers: headers, - filename: "Clue_Mediator_Report.csv", - }; + const [csvLinkEl, setCsvLinkEl] = useState(React.createRef()); //Testdata to Be ignored const downloadReport = async () => { - setData(await ResourcesService.GetReport(executionId, filters)); //tobefixed + const tempData = await ResourcesService.GetReport( + currentExecution, + filters + ).catch(() => false); + + if (tempData) { + setData(tempData); + } + csvLinkEl.current.link.click(); }; + return (
{ />
); }; -//Todo -// -> Get the Execution ID and pass it do downloadReport - +CSVDownloadButton.defaultProps = {}; CSVDownloadButton.propTypes = { - executionId: PropTypes.string, + currentExecution: PropTypes.string, filters: PropTypes.array, }; const mapStateToProps = (state) => ({ - executionId: state.executions.current, + currentExecution: state.executions.current, filters: state.filters.filters, }); diff --git a/ui/src/components/Dashboard/ResourcesChart.js b/ui/src/components/Dashboard/ResourcesChart.js index 77d60d8e..345cc68c 100644 --- a/ui/src/components/Dashboard/ResourcesChart.js +++ b/ui/src/components/Dashboard/ResourcesChart.js @@ -187,7 +187,7 @@ const ResourcesChart = ({ {!isResourceListLoading && sortedResources.length > 0 && ( - +

{account ? `${accounts[account].Name} (${accounts[account].ID}):` diff --git a/ui/src/services/report.service.js b/ui/src/services/report.service.js deleted file mode 100644 index f75b8630..00000000 --- a/ui/src/services/report.service.js +++ /dev/null @@ -1,18 +0,0 @@ -import { http } from "./request.service"; - -export const AccsService = { - fetchSummary, -}; - -/** - * - * @param {string} executionId execution to query - */ -function fetchSummary(executionId) { - return http - .send(`api/v1/getReport/${executionId}`, `get`) - .then(this.handleResponse) - .then((response) => { - return response; - }); -} diff --git a/ui/src/services/resources.service.js b/ui/src/services/resources.service.js index 98a9aba6..b2513bcb 100644 --- a/ui/src/services/resources.service.js +++ b/ui/src/services/resources.service.js @@ -13,7 +13,6 @@ export const ResourcesService = { * @returns filters params for request */ const getTransformedFilters = (filters) => { - console.log(filters); const params = {}; filters.forEach((filter) => { if (filter.type === "resource") { @@ -91,7 +90,7 @@ function GetContent(name, executionID, filters = []) { }); } -function GetReport(executionId, filters) { +function GetReport(executionID, filters = []) { const params = { ...getTransformedFilters(filters), }; @@ -99,7 +98,7 @@ function GetReport(executionId, filters) { new window.URLSearchParams(params).toString() ); return http - .send(`api/v1/getReport/${executionId}?${searchParams}`, `get`) + .send(`api/v1/report/${executionID}?${searchParams}`, `get`) .then(this.handleResponse) .then((response) => { return response; From 6f8282086aee4cc4cbba6da0ff1132192abb2e19 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Mon, 28 Jun 2021 14:20:02 +0200 Subject: [PATCH 29/68] #SOS-33 add S3 ("simple storage service") checks and test-file change cloudwatch.go return error if no datapoint is returned from cloudwatch --- collector/aws/cloudwatch/cloudwatch.go | 5 +- collector/aws/resources/s3.go | 196 +++++++++++++++++++++++++ collector/aws/resources/s3_test.go | 145 ++++++++++++++++++ 3 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 collector/aws/resources/s3.go create mode 100644 collector/aws/resources/s3_test.go diff --git a/collector/aws/cloudwatch/cloudwatch.go b/collector/aws/cloudwatch/cloudwatch.go index 0317d6e4..8b02fca7 100644 --- a/collector/aws/cloudwatch/cloudwatch.go +++ b/collector/aws/cloudwatch/cloudwatch.go @@ -50,6 +50,10 @@ func (cw *CloudwatchManager) GetMetric(metricInput *awsCloudwatch.GetMetricStati return calculatedMetricValue, metricsResponseValue, err } + if len(metricData.Datapoints) == 0 { + return calculatedMetricValue, metricsResponseValue, errors.New("No Datapoints in this region for this resource.") + } + switch metric.Statistic { case "Average": calculatedMetricValue = cw.AvgDatapoint(metricData) @@ -77,7 +81,6 @@ func (cw *CloudwatchManager) GetMetric(metricInput *awsCloudwatch.GetMetricStati if err != nil { return calculatedMetricValue, metricsResponseValue, err } - return formulaResponse.(float64), metricsResponseValue, nil } diff --git a/collector/aws/resources/s3.go b/collector/aws/resources/s3.go new file mode 100644 index 00000000..4ca5dd92 --- /dev/null +++ b/collector/aws/resources/s3.go @@ -0,0 +1,196 @@ +package resources + +import ( + "errors" + "finala/collector" + "finala/collector/aws/common" + "finala/collector/aws/register" + "finala/collector/config" + "finala/expression" + awsClient "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/arn" + awsCloudwatch "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/s3" + log "github.com/sirupsen/logrus" + "time" +) + +// S3ClientDescriptor is an interface defining the aws s3 client +type S3ClientDescriptor interface { + ListBuckets(input *s3.ListBucketsInput) (*s3.ListBucketsOutput, error) +} + +// S3Manager describes S3 struct +type S3Manager struct { + client S3ClientDescriptor + awsManager common.AWSManager + namespace string + servicePricingCode string + Name collector.ResourceIdentifier +} + +// DetectedS3 define the detected AWS S3 +type DetectedS3 struct { + Region string + Metric string + Name string + ResourceID string + LaunchTime time.Time + collector.AccountSpecifiedFields +} + +func init() { + register.Registry("s3", NewS3Manager) +} + +// NewS3Manager implements AWS GO SDK +func NewS3Manager(awsManager common.AWSManager, client interface{}) (common.ResourceDetection, error) { + + if client == nil { + client = s3.New(awsManager.GetSession()) + } + + s3Client, ok := client.(S3ClientDescriptor) + if !ok { + return nil, errors.New("invalid s3 client") + } + + return &S3Manager{ + client: s3Client, + awsManager: awsManager, + namespace: "AWS/S3", + servicePricingCode: "AmazonS3", + Name: awsManager.GetResourceIdentifier("s3"), + }, nil + +} + +// Detect S3 buckets is under utilized +func (s *S3Manager) Detect(metrics []config.MetricConfig) (interface{}, error) { + + log.WithFields(log.Fields{ + "region": s.awsManager.GetRegion(), + "resource": "s3_buckets", + }).Info("starting to analyze resource") + + s.awsManager.GetCollector().CollectStart(s.Name, collector.AccountSpecifiedFields{ + AccountID: *s.awsManager.GetAccountIdentity().Account, + AccountName: s.awsManager.GetAccountName(), + }) + + detectedS3 := []DetectedS3{} + + buckets, err := s.listBuckets(nil) + if err != nil { + s.awsManager.GetCollector().CollectError(s.Name, err) + return detectedS3, err + } + now := time.Now() + + for _, bucket := range buckets { + log.WithField("bucket_name", *bucket.Name).Debug("checking s3 bucket") + + for _, metric := range metrics { + + period := int64(metric.Period.Seconds()) + metricEndTime := now.Add(time.Duration(-metric.StartTime)) + metricInput := awsCloudwatch.GetMetricStatisticsInput{ + Namespace: &s.namespace, + MetricName: &metric.Description, + Period: &period, + StartTime: &metricEndTime, + EndTime: &now, + Dimensions: []*awsCloudwatch.Dimension{ + { + Name: awsClient.String("BucketName"), + Value: bucket.Name, + }, + { + Name: awsClient.String("FilterId"), + Value: awsClient.String("EntireBucket"), + }, + }, + } + + formulaValue, _, err := s.awsManager.GetCloudWatchClient().GetMetric(&metricInput, metric) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "bucket_name": *bucket.Name, + "metric_name": metric.Description, + }).Error("Could not get cloudwatch metric data") + continue + } + + expression, err := expression.BoolExpression(formulaValue, metric.Constraint.Value, metric.Constraint.Operator) + if err != nil || formulaValue == float64(-1) { + log.Info("formel == -1") + continue + } + + if expression { + + log.WithFields(log.Fields{ + "metric_name": metric.Description, + "constraint_operator": metric.Constraint.Operator, + "constraint_Value": metric.Constraint.Value, + "formula_value": formulaValue, + "bucket_name": *bucket.Name, + "region": s.awsManager.GetRegion(), + }).Info("EC2 instance detected as unutilized resource") + + Arn := "arn:aws:s3:::" + *bucket.Name + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + + s3 := DetectedS3{ + Region: s.awsManager.GetRegion(), + Metric: metric.Description, + Name: *bucket.Name, + ResourceID: Arn, + LaunchTime: *bucket.CreationDate, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *s.awsManager.GetAccountIdentity().Account, + AccountName: s.awsManager.GetAccountName(), + }, + } + + s.awsManager.GetCollector().AddResource(collector.EventCollector{ + ResourceName: s.Name, + Data: s3, + }) + + detectedS3 = append(detectedS3, s3) + } + } + } + + s.awsManager.GetCollector().CollectFinish(s.Name, collector.AccountSpecifiedFields{ + AccountID: *s.awsManager.GetAccountIdentity().Account, + AccountName: s.awsManager.GetAccountName(), + }) + + return detectedS3, nil +} + +func (s *S3Manager) listBuckets(buckets []*s3.Bucket) ([]*s3.Bucket, error) { + + input := &s3.ListBucketsInput{} + + resp, err := s.client.ListBuckets(input) + if err != nil { + log.WithField("error", err).Error("could not list s3 buckets") + return nil, err + } + + if buckets == nil { + buckets = []*s3.Bucket{} + } + + buckets = append(buckets, resp.Buckets...) + + return buckets, nil +} diff --git a/collector/aws/resources/s3_test.go b/collector/aws/resources/s3_test.go new file mode 100644 index 00000000..7ee9ec5c --- /dev/null +++ b/collector/aws/resources/s3_test.go @@ -0,0 +1,145 @@ +package resources + +import ( + "errors" + awsTestutils "finala/collector/aws/testutils" + "finala/collector/config" + "finala/collector/testutils" + collectorTestutils "finala/collector/testutils" + awsClient "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "reflect" + "testing" + "time" +) + +var defaultS3Mock = s3.ListBucketsOutput{ + Buckets: []*s3.Bucket{ + { + Name: awsClient.String("test-bucket"), + CreationDate: testutils.TimePointer(time.Now()), + }, + }, +} + +type MockAWSS3Client struct { + responseListBuckets s3.ListBucketsOutput + err error +} + +func (r *MockAWSS3Client) ListBuckets(input *s3.ListBucketsInput) (*s3.ListBucketsOutput, error) { + return &r.responseListBuckets, r.err +} + +func TestS3ListBuckets(t *testing.T) { + + collector := collectorTestutils.NewMockCollector() + detector := awsTestutils.AWSManager(collector, nil, nil, "us-east-1") + + t.Run("valid", func(t *testing.T) { + + mockClient := MockAWSS3Client{ + responseListBuckets: defaultS3Mock, + } + + s3Interface, err := NewS3Manager(detector, &mockClient) + if err != nil { + t.Fatalf("unexpected s3 manager error happend, got %v expected %v", err, nil) + } + + s3Manager, ok := s3Interface.(*S3Manager) + if !ok { + t.Fatalf("unexpected s3 struct, got %s expected %s", reflect.TypeOf(s3Interface), "*S3Manager") + } + + result, _ := s3Manager.listBuckets(nil) + + if len(result) != len(defaultS3Mock.Buckets) { + t.Fatalf("unexpected S3 bucket count, got %d expected %d", len(result), len(defaultS3Mock.Buckets)) + } + + }) + + t.Run("error", func(t *testing.T) { + mockClient := MockAWSS3Client{ + responseListBuckets: defaultS3Mock, + err: errors.New("error"), + } + + s3, err := NewS3Manager(detector, &mockClient) + if err != nil { + t.Fatalf("unexpected s3 manager error happend, got %v expected %v", err, nil) + } + + s3Manager, ok := s3.(*S3Manager) + if !ok { + t.Fatalf("unexpected s3 struct, got %s expected %s", reflect.TypeOf(s3Manager), "*S3Manager") + } + + _, err = s3Manager.listBuckets(nil) + + if err == nil { + t.Fatalf("unexpected list buckets error, return empty") + } + }) + +} + +func TestDetectS3(t *testing.T) { + + collector := collectorTestutils.NewMockCollector() + mockCloudwatch := awsTestutils.NewMockCloudwatch(nil) + detector := awsTestutils.AWSManager(collector, mockCloudwatch, nil, "us-east-1") + + var defaultMetricConfig = []config.MetricConfig{ + { + Description: "test description write capacity", + Data: []config.MetricDataConfiguration{ + { + Name: "TestMetric", + Statistic: "Sum", + }, + }, + Constraint: config.MetricConstraintConfig{ + Operator: "==", + Value: 5, + }, + Period: 1, + StartTime: 1, + }, + } + + mockClient := MockAWSS3Client{ + responseListBuckets: defaultS3Mock, + } + + s3, err := NewS3Manager(detector, &mockClient) + if err != nil { + t.Fatalf("unexpected s3 manager error happend, got %v expected %v", err, nil) + } + + s3Manager, ok := s3.(*S3Manager) + if !ok { + t.Fatalf("unexpected s3 struct, got %s expected %s", reflect.TypeOf(s3Manager), "*S3Manager") + } + + response, _ := s3Manager.Detect(defaultMetricConfig) + + s3Response, ok := response.([]DetectedS3) + if !ok { + t.Fatalf("unexpected s3 struct, got %s expected %s", reflect.TypeOf(response), "[]DetectedS3") + } + + if len(s3Response) != 1 { + t.Fatalf("unexpected s3 detected, got %d expected %d", len(s3Response), 1) + } + + if len(collector.Events) != 1 { + t.Fatalf("unexpected collector s3 resources, got %d expected %d", len(collector.Events), 1) + } + + if len(collector.EventsCollectionStatus) != 2 { + t.Fatalf("unexpected resource status events count, got %d expected %d", len(collector.EventsCollectionStatus), 2) + } + +} From 580c0fc943fefddeb1e210853cb7b38f440d6e48 Mon Sep 17 00:00:00 2001 From: DerOtt Date: Mon, 28 Jun 2021 15:46:43 +0200 Subject: [PATCH 30/68] CSVDownloadButton now only shows up in 'Summary' ResourceChart to avoid two or even more buttons to show up Added an extra check if element exists and is not nil in GetReport in API Backend as in edge cases getResources can deliver empty result sets. --- api/route.go | 6 +++++- ui/src/components/Dashboard/CSVDownloadButton.js | 6 +++++- ui/src/components/Dashboard/ResourcesChart.js | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/api/route.go b/api/route.go index 4dd929c3..c29d5c85 100644 --- a/api/route.go +++ b/api/route.go @@ -232,7 +232,6 @@ func (server *Server) GetReport(resp http.ResponseWriter, req *http.Request) { fmt.Println(resourceName, resourceSummary) //sanity test if resourceSummary.ResourceCount > 0 { resourcesList, err := server.storage.GetResources(resourceName, executionID, filters) - if err != nil { server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) continue @@ -244,6 +243,11 @@ func (server *Server) GetReport(resp http.ResponseWriter, req *http.Request) { "filter": filters, }).Info(resourcesList) + //can still fail for some odd reason so we check if the resource is actually there + if resourcesList[0] == nil { + server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) + return + } data, ok := resourcesList[0]["Data"].(map[string]interface{}) if !ok { //screw your log diff --git a/ui/src/components/Dashboard/CSVDownloadButton.js b/ui/src/components/Dashboard/CSVDownloadButton.js index e7f52e9d..58ad3bbf 100644 --- a/ui/src/components/Dashboard/CSVDownloadButton.js +++ b/ui/src/components/Dashboard/CSVDownloadButton.js @@ -8,7 +8,11 @@ const CSVDownloadButton = ({ currentExecution, filters }) => { const [data, setData] = useState([]); const [csvLinkEl, setCsvLinkEl] = useState(React.createRef()); - //Testdata to Be ignored + //Todo + // Button muss in summary chart bereich und nur wenn daten (resourcechart.js) erkennbar wenn && !accountid gesetzt DONE + // ArrayIndexOutofBounds Fehler fixen / bzw schauen ob er noch passieren kann (zweites if arr[0] leer dann abfangen) + // Style optional anpassen + // Veraenderungen in richtigen branch setzen (merge) const downloadReport = async () => { const tempData = await ResourcesService.GetReport( diff --git a/ui/src/components/Dashboard/ResourcesChart.js b/ui/src/components/Dashboard/ResourcesChart.js index 345cc68c..73648d38 100644 --- a/ui/src/components/Dashboard/ResourcesChart.js +++ b/ui/src/components/Dashboard/ResourcesChart.js @@ -187,7 +187,7 @@ const ResourcesChart = ({ {!isResourceListLoading && sortedResources.length > 0 && ( - + {!account && }

{account ? `${accounts[account].Name} (${accounts[account].ID}):` From e0a87bd0c9a4dca8d3e8df4b8864c2ba86488b67 Mon Sep 17 00:00:00 2001 From: DerOtt Date: Tue, 29 Jun 2021 17:15:27 +0200 Subject: [PATCH 31/68] Removed exactly one whitespace --- ui/src/components/Dashboard/ResourcesChart.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/components/Dashboard/ResourcesChart.js b/ui/src/components/Dashboard/ResourcesChart.js index 73648d38..7fab910b 100644 --- a/ui/src/components/Dashboard/ResourcesChart.js +++ b/ui/src/components/Dashboard/ResourcesChart.js @@ -187,7 +187,7 @@ const ResourcesChart = ({ {!isResourceListLoading && sortedResources.length > 0 && ( - {!account && } + {!account && }

{account ? `${accounts[account].Name} (${accounts[account].ID}):` From eeebb64a00a3dbc58ddbf84e8a2230ac84ff6483 Mon Sep 17 00:00:00 2001 From: DerOtt Date: Tue, 29 Jun 2021 17:20:30 +0200 Subject: [PATCH 32/68] Added exactly one whitespace --- ui/src/components/Dashboard/ResourcesChart.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/components/Dashboard/ResourcesChart.js b/ui/src/components/Dashboard/ResourcesChart.js index 7fab910b..664bf370 100644 --- a/ui/src/components/Dashboard/ResourcesChart.js +++ b/ui/src/components/Dashboard/ResourcesChart.js @@ -187,7 +187,7 @@ const ResourcesChart = ({ {!isResourceListLoading && sortedResources.length > 0 && ( - {!account && } + {!account && }

{account ? `${accounts[account].Name} (${accounts[account].ID}):` From f380430253d2d075009a42a076d8b01c35d6a847 Mon Sep 17 00:00:00 2001 From: DerOtt Date: Wed, 30 Jun 2021 14:29:11 +0200 Subject: [PATCH 33/68] Removed absolutely nothing --- .../components/Dashboard/CSVDownloadButton.js | 26 ++++++++++++++----- ui/src/components/Dashboard/ResourcesChart.js | 2 +- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/ui/src/components/Dashboard/CSVDownloadButton.js b/ui/src/components/Dashboard/CSVDownloadButton.js index 58ad3bbf..71355cb8 100644 --- a/ui/src/components/Dashboard/CSVDownloadButton.js +++ b/ui/src/components/Dashboard/CSVDownloadButton.js @@ -3,16 +3,27 @@ import { CSVLink } from "react-csv"; import { ResourcesService } from "services/resources.service"; import { connect } from "react-redux"; import PropTypes from "prop-types"; +import { makeStyles } from "@material-ui/core/styles"; + +const useStyles = makeStyles(() => ({ + myButton: { + backgroundColor: "#d5dee6", + borderColor: "#d5dee6", + color: "rgba(0, 0, 0, 0.87)", + border: "0", + textAlign: "center", + fontWeight: "bold", + width: "100%", + maxWidth: "320px", + float: "right", + borderRadius: "4px", + }, +})); const CSVDownloadButton = ({ currentExecution, filters }) => { const [data, setData] = useState([]); const [csvLinkEl, setCsvLinkEl] = useState(React.createRef()); - - //Todo - // Button muss in summary chart bereich und nur wenn daten (resourcechart.js) erkennbar wenn && !accountid gesetzt DONE - // ArrayIndexOutofBounds Fehler fixen / bzw schauen ob er noch passieren kann (zweites if arr[0] leer dann abfangen) - // Style optional anpassen - // Veraenderungen in richtigen branch setzen (merge) + const classes = useStyles(); const downloadReport = async () => { const tempData = await ResourcesService.GetReport( @@ -29,13 +40,14 @@ const CSVDownloadButton = ({ currentExecution, filters }) => { return (
diff --git a/ui/src/components/Dashboard/ResourcesChart.js b/ui/src/components/Dashboard/ResourcesChart.js index 664bf370..26c7f824 100644 --- a/ui/src/components/Dashboard/ResourcesChart.js +++ b/ui/src/components/Dashboard/ResourcesChart.js @@ -187,12 +187,12 @@ const ResourcesChart = ({ {!isResourceListLoading && sortedResources.length > 0 && ( - {!account && }

{account ? `${accounts[account].Name} (${accounts[account].ID}):` : "Summary:"}

+ {!account && } Date: Wed, 30 Jun 2021 16:50:21 +0200 Subject: [PATCH 34/68] bugfix: Kinesis priceList error bugfix: ElasticIP priceList error bugfix: DocumentDB engineName error bugfix: DocumentDB priceList error --- collector/aws/pricing/pricing.go | 2 +- collector/aws/resources/docdb.go | 11 +++-- collector/aws/resources/elasticips.go | 63 ++++++++++++++++++--------- collector/aws/resources/kinesis.go | 15 +++++++ 4 files changed, 63 insertions(+), 28 deletions(-) diff --git a/collector/aws/pricing/pricing.go b/collector/aws/pricing/pricing.go index 60863a28..74d2127e 100644 --- a/collector/aws/pricing/pricing.go +++ b/collector/aws/pricing/pricing.go @@ -45,7 +45,7 @@ var regionsInfo = map[string]regionInfo{ "cn-north-1": {fullName: "China (Beijing)", prefix: ""}, "cn-northwest-1": {fullName: "China (Ningxia)", prefix: ""}, "eu-central-1": {fullName: "EU (Frankfurt)", prefix: "EUC1"}, - "eu-west-1": {fullName: "EU (Ireland)", prefix: "EUW1"}, + "eu-west-1": {fullName: "EU (Ireland)", prefix: "EU"}, "eu-west-2": {fullName: "EU (London)", prefix: "EUW2"}, "eu-west-3": {fullName: "EU (Paris)", prefix: "EUW3"}, "eu-south-1": {fullName: "EU (Milan)", prefix: "EUS1"}, diff --git a/collector/aws/resources/docdb.go b/collector/aws/resources/docdb.go index 1929dde3..a01fe263 100644 --- a/collector/aws/resources/docdb.go +++ b/collector/aws/resources/docdb.go @@ -209,6 +209,11 @@ func (dd *DocumentDBManager) getPricingFilterInput(instance *docdb.DBInstance) p Field: awsClient.String("instanceType"), Value: instance.DBInstanceClass, }, + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("productFamily"), + Value: awsClient.String("Database Instance"), + }, }, } } @@ -218,12 +223,6 @@ func (dd *DocumentDBManager) describeInstances(marker *string, instances []*docd input := &docdb.DescribeDBInstancesInput{ Marker: marker, - Filters: []*docdb.Filter{ - { - Name: awsClient.String("engine"), - Values: []*string{awsClient.String("docdb")}, - }, - }, } resp, err := dd.client.DescribeDBInstances(input) diff --git a/collector/aws/resources/elasticips.go b/collector/aws/resources/elasticips.go index dbc7d8a0..d02aa8c3 100644 --- a/collector/aws/resources/elasticips.go +++ b/collector/aws/resources/elasticips.go @@ -6,6 +6,7 @@ import ( "finala/collector/aws/common" "finala/collector/aws/register" "finala/collector/config" + "fmt" "github.com/aws/aws-sdk-go/aws/arn" awsClient "github.com/aws/aws-sdk-go/aws" @@ -79,14 +80,28 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, elasticIPs := []DetectedElasticIP{} - priceFIlters := ei.getPricingFilterInput() + pricingRegionPrefix, err := ei.awsManager.GetPricingClient().GetRegionPrefix(ei.awsManager.GetRegion()) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "region": ei.awsManager.GetRegion(), + }).Error("Could not get pricing region prefix") + ei.awsManager.GetCollector().CollectError(ei.Name, err) + return elasticIPs, err + } + + priceFilters := ei.getPricingFilterInput([]*pricing.Filter{ + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("usagetype"), + Value: awsClient.String(fmt.Sprintf("%sElasticIP:IdleAddress", pricingRegionPrefix)), + }}) // Get elastic ip pricing - price, err := ei.awsManager.GetPricingClient().GetPrice(priceFIlters, ei.rateCode, ei.awsManager.GetRegion()) + price, err := ei.awsManager.GetPricingClient().GetPrice(priceFilters, ei.rateCode, ei.awsManager.GetRegion()) if err != nil { log.WithError(err).WithFields(log.Fields{ "rate_code": ei.rateCode, "region": ei.awsManager.GetRegion(), - "price_filters": priceFIlters, + "price_filters": priceFilters, }).Error("could not get elastic ip price") ei.awsManager.GetCollector().CollectError(ei.Name, err) @@ -155,27 +170,33 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, } // getPricingFilterInput returns the elastic ip price filters. -func (ei *ElasticIPManager) getPricingFilterInput() pricing.GetProductsInput { +func (ei *ElasticIPManager) getPricingFilterInput(extraFilters []*pricing.Filter) pricing.GetProductsInput { + + filters := []*pricing.Filter{ + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("TermType"), + Value: awsClient.String("OnDemand"), + }, + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("productFamily"), + Value: awsClient.String("IP Address"), + }, + { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("group"), + Value: awsClient.String("ElasticIP:Address"), + }, + } + + if extraFilters != nil { + filters = append(filters, extraFilters...) + } return pricing.GetProductsInput{ ServiceCode: &ei.servicePricingCode, - Filters: []*pricing.Filter{ - { - Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("TermType"), - Value: awsClient.String("OnDemand"), - }, - { - Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("productFamily"), - Value: awsClient.String("IP Address"), - }, - { - Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("group"), - Value: awsClient.String("ElasticIP:Address"), - }, - }, + Filters: filters, } } diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index 16c6e571..c269e82c 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "fmt" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -98,6 +99,16 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er log.WithError(err).Error("Could not get shard price") return detectedStreams, err } + + pricingRegionPrefix, err := km.awsManager.GetPricingClient().GetRegionPrefix(km.awsManager.GetRegion()) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "region": km.awsManager.GetRegion(), + }).Error("Could not get pricing region prefix") + km.awsManager.GetCollector().CollectError(km.Name, err) + return detectedStreams, err + } + // Get Price for extended Shard Hour retention extendedRetentionPrice, err := km.awsManager.GetPricingClient().GetPrice( km.getPricingFilterInput([]*pricing.Filter{ @@ -105,6 +116,10 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er Type: awsClient.String("TERM_MATCH"), Field: awsClient.String("group"), Value: awsClient.String("Addon shard hour"), + }, { + Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("usageType"), + Value: awsClient.String(fmt.Sprintf("%sLoadBalancerUsage", pricingRegionPrefix)), }}), "", km.awsManager.GetRegion()) if err != nil { log.WithError(err).Error("Could not get shard extended retention price") From c6b916e94b6fe6148b021380bb472cc7448b5c1f Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 30 Jun 2021 17:20:26 +0200 Subject: [PATCH 35/68] bugfix: kinesis priceList Error --- collector/aws/resources/kinesis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index c269e82c..6408f6fe 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -119,7 +119,7 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er }, { Type: awsClient.String("TERM_MATCH"), Field: awsClient.String("usageType"), - Value: awsClient.String(fmt.Sprintf("%sLoadBalancerUsage", pricingRegionPrefix)), + Value: awsClient.String(fmt.Sprintf("%sStorage-ShardHour", pricingRegionPrefix)), }}), "", km.awsManager.GetRegion()) if err != nil { log.WithError(err).Error("Could not get shard extended retention price") From 9242b0ab9a63dee20028b429462d13c62082a8e5 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 30 Jun 2021 17:36:54 +0200 Subject: [PATCH 36/68] bugfix: kinesis priceList error bugfix: elasticIP priceList error --- collector/aws/resources/elasticips.go | 5 ----- collector/aws/resources/kinesis.go | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/collector/aws/resources/elasticips.go b/collector/aws/resources/elasticips.go index d02aa8c3..67546b36 100644 --- a/collector/aws/resources/elasticips.go +++ b/collector/aws/resources/elasticips.go @@ -183,11 +183,6 @@ func (ei *ElasticIPManager) getPricingFilterInput(extraFilters []*pricing.Filter Field: awsClient.String("productFamily"), Value: awsClient.String("IP Address"), }, - { - Type: awsClient.String("TERM_MATCH"), - Field: awsClient.String("group"), - Value: awsClient.String("ElasticIP:Address"), - }, } if extraFilters != nil { diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index 6408f6fe..28ff906e 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -119,7 +119,7 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er }, { Type: awsClient.String("TERM_MATCH"), Field: awsClient.String("usageType"), - Value: awsClient.String(fmt.Sprintf("%sStorage-ShardHour", pricingRegionPrefix)), + Value: awsClient.String(fmt.Sprintf("%sExtended-ShardHour", pricingRegionPrefix)), }}), "", km.awsManager.GetRegion()) if err != nil { log.WithError(err).Error("Could not get shard extended retention price") From 0912705009bb44596924ea4f8b274f5c1f9dfeba Mon Sep 17 00:00:00 2001 From: DerOtt Date: Thu, 1 Jul 2021 13:52:40 +0200 Subject: [PATCH 37/68] Testing --- ui/src/components/Dashboard/CSVDownloadButton.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ui/src/components/Dashboard/CSVDownloadButton.js b/ui/src/components/Dashboard/CSVDownloadButton.js index 71355cb8..9adc282d 100644 --- a/ui/src/components/Dashboard/CSVDownloadButton.js +++ b/ui/src/components/Dashboard/CSVDownloadButton.js @@ -4,6 +4,7 @@ import { ResourcesService } from "services/resources.service"; import { connect } from "react-redux"; import PropTypes from "prop-types"; import { makeStyles } from "@material-ui/core/styles"; +import { Button } from "@material-ui/core/Button" const useStyles = makeStyles(() => ({ myButton: { @@ -45,6 +46,12 @@ const CSVDownloadButton = ({ currentExecution, filters }) => { value="Download Current Data as CSV Report" onClick={downloadReport} /> + {!isResourceListLoading && sortedResources.length > 0 && ( + {!account && }

{account ? `${accounts[account].Name} (${accounts[account].ID}):` : "Summary:"}

- {!account && } Date: Thu, 1 Jul 2021 17:39:53 +0200 Subject: [PATCH 42/68] #SOS-29 --- configuration/collector.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/configuration/collector.yaml b/configuration/collector.yaml index 95557126..d7ee1483 100644 --- a/configuration/collector.yaml +++ b/configuration/collector.yaml @@ -200,6 +200,17 @@ providers: constraint: operator: "==" value: 0 + ecs: + - description: Percentage of used CPU + enable: true + metrics: + - name: CPUUtilization + statistic: Average + period: 24h + start_time: 168h # 24h * 7d + constraint: + operator: "<" + value: 5 natgateway: - description: Active connection count enable: true From 4df494c82572853126641ba5ca3e394c62f97f7d Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Fri, 2 Jul 2021 16:01:02 +0200 Subject: [PATCH 43/68] fix warnings: unused statements --- ui/src/components/Dashboard/Index.js | 1 - ui/src/services/resources.service.js | 1 - 2 files changed, 2 deletions(-) diff --git a/ui/src/components/Dashboard/Index.js b/ui/src/components/Dashboard/Index.js index cb963357..5d48e9b8 100644 --- a/ui/src/components/Dashboard/Index.js +++ b/ui/src/components/Dashboard/Index.js @@ -8,7 +8,6 @@ import AccountsList from "./AccountsList"; import FilterBar from "./FilterBar"; import StatisticsBar from "./StatisticsBar"; import ResourceScanning from "./ResourceScanning"; -import ResourcesChart from "./ResourcesChart"; import ResourcesCharts from "./ResourcesCharts"; import ResourcesList from "./ResourcesList"; import ResourceTable from "./ResourceTable"; diff --git a/ui/src/services/resources.service.js b/ui/src/services/resources.service.js index fa4604a6..8ab79481 100644 --- a/ui/src/services/resources.service.js +++ b/ui/src/services/resources.service.js @@ -12,7 +12,6 @@ export const ResourcesService = { * @returns filters params for request */ const getTransformedFilters = (filters) => { - console.log(filters); const params = {}; filters.forEach((filter) => { if (filter.type === "resource") { From 8e464cfd4c34ce636bd9bcc6b2554d3e286b0427 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Fri, 2 Jul 2021 16:02:24 +0200 Subject: [PATCH 44/68] fix warnings: unused statements --- ui/src/components/Dashboard/CSVDownloadButton.js | 2 +- ui/src/components/Dashboard/Index.js | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ui/src/components/Dashboard/CSVDownloadButton.js b/ui/src/components/Dashboard/CSVDownloadButton.js index 1d393952..56b47e03 100644 --- a/ui/src/components/Dashboard/CSVDownloadButton.js +++ b/ui/src/components/Dashboard/CSVDownloadButton.js @@ -22,7 +22,7 @@ const useStyles = makeStyles(() => ({ const CSVDownloadButton = ({ currentExecution, filters }) => { const [data, setData] = useState([]); - const [csvLinkEl, setCsvLinkEl] = useState(React.createRef()); + const [csvLinkEl] = useState(React.createRef()); const classes = useStyles(); const downloadReport = async () => { diff --git a/ui/src/components/Dashboard/Index.js b/ui/src/components/Dashboard/Index.js index cb963357..5d48e9b8 100644 --- a/ui/src/components/Dashboard/Index.js +++ b/ui/src/components/Dashboard/Index.js @@ -8,7 +8,6 @@ import AccountsList from "./AccountsList"; import FilterBar from "./FilterBar"; import StatisticsBar from "./StatisticsBar"; import ResourceScanning from "./ResourceScanning"; -import ResourcesChart from "./ResourcesChart"; import ResourcesCharts from "./ResourcesCharts"; import ResourcesList from "./ResourcesList"; import ResourceTable from "./ResourceTable"; From c4876ef6b3601ead8749fbf7197a1afd82b54960 Mon Sep 17 00:00:00 2001 From: DerOtt Date: Tue, 6 Jul 2021 15:36:13 +0200 Subject: [PATCH 45/68] Testing --- collector/aws/resources/ecs.go | 240 +++++++++++++++++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 collector/aws/resources/ecs.go diff --git a/collector/aws/resources/ecs.go b/collector/aws/resources/ecs.go new file mode 100644 index 00000000..c2896555 --- /dev/null +++ b/collector/aws/resources/ecs.go @@ -0,0 +1,240 @@ +package resources + +import ( + "errors" + "finala/collector" + "finala/collector/aws/common" + "finala/collector/aws/register" + "finala/collector/config" + "finala/expression" + awsClient "github.com/aws/aws-sdk-go/aws" + awsCloudwatch "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/ecs" + log "github.com/sirupsen/logrus" + "strings" + "time" +) + +type EcsClientDescriptor interface { + DescribeServices(input *ecs.DescribeServicesInput) (*ecs.DescribeServicesOutput, error) + ListClusters(input *ecs.ListClustersInput) (*ecs.ListClustersOutput, error) + ListServices(input *ecs.ListServicesInput) (*ecs.ListServicesOutput, error) +} + +type EcsManager struct { + client EcsClientDescriptor + awsManager common.AWSManager + namespace string + Name collector.ResourceIdentifier +} + +type DetectedEcs struct { + Region string + Metric string + LaunchType string + collector.AccountSpecifiedFields + collector.PriceDetectedFields +} + +func init() { + register.Registry("ecs", NewEcsManager) +} + +func NewEcsManager(awsManager common.AWSManager, client interface{}) (common.ResourceDetection, error) { + if client == nil { + client = ecs.New(awsManager.GetSession()) + } + + ecsClient, ok := client.(EcsClientDescriptor) + if !ok { + return nil, errors.New("invalid ecs client") + } + + return &EcsManager{ + client: ecsClient, + awsManager: awsManager, + namespace: "AWS/ECS", + Name: awsManager.GetResourceIdentifier("ECS"), + }, nil +} + +func (ec *EcsManager) Detect(metrics []config.MetricConfig) (interface{}, error) { + log.WithFields(log.Fields{ + "region": ec.awsManager.GetRegion(), + "resource": "ecs", + }).Info("starting to analyze resource") + + ec.awsManager.GetCollector().CollectStart(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) + + detectedEcsServices := []DetectedEcs{} + + services, err := ec.describeServices(nil, nil) + if err != nil { + ec.awsManager.GetCollector().CollectError(ec.Name, err) + return detectedEcsServices, err + } + + now := time.Now() + + for _, service := range services { + log.WithField("service_name", *service.ServiceName).Debug("checking ecs") + + for _, metric := range metrics { + log.WithFields(log.Fields{ + "service_name": *service.ServiceName, + "metric_name": metric.Description, + }).Debug("checking metric") + + period := int64(metric.Period.Seconds()) + metricEndTime := now.Add(time.Duration(-metric.StartTime)) + tmp := strings.Split(*service.ServiceArn, "/") + metricInput := awsCloudwatch.GetMetricStatisticsInput{ + Dimensions: []*awsCloudwatch.Dimension{ + { + Name: awsClient.String("ServiceName"), + Value: service.ServiceName, + }, { + Name: awsClient.String("ClusterName"), + Value: &(tmp[len(tmp)-1]), + }, + }, + EndTime: &now, + ExtendedStatistics: nil, + MetricName: &metric.Description, + Namespace: &ec.namespace, + Period: &period, + StartTime: &metricEndTime, + Statistics: nil, + Unit: nil, + } + + formulaValue, _, err := ec.awsManager.GetCloudWatchClient().GetMetric(&metricInput, metric) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "service_name": *service.ServiceName, + "metric_name": metric.Description, + }).Error("Could not get cloudwatch metric data") + continue + } + + expression, err := expression.BoolExpression(formulaValue, metric.Constraint.Value, metric.Constraint.Operator) + if err != nil { + log.WithField("error", err).Error("could not parse expression") + continue + } + + if expression { + log.WithFields(log.Fields{ + "metric_name": metric.Description, + "constraint_operator": metric.Constraint.Operator, + "constraint_Value": metric.Constraint.Value, + "formula_value": formulaValue, + "service_name": *service.ServiceName, + "region": ec.awsManager.GetRegion(), + "launch_type": service.LaunchType, + }).Info("Redshift cluster detected as unutilized resource") + + tagsData := map[string]string{} + if err == nil { + for _, tag := range service.Tags { + tagsData[*tag.Key] = *tag.Value + } + } + + ecss := DetectedEcs{ + Region: ec.awsManager.GetRegion(), + Metric: metric.Description, + LaunchType: *service.LaunchType, + PriceDetectedFields: collector.PriceDetectedFields{ + ResourceID: *service.ServiceArn, + LaunchTime: *service.CreatedAt, + Tag: tagsData, + }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }, + } + ec.awsManager.GetCollector().AddResource(collector.EventCollector{ + ResourceName: collector.ResourceIdentifier(ec.Name), + Data: ecss, + }) + + detectedEcsServices = append(detectedEcsServices, ecss) + + } + + } + } + + ec.awsManager.GetCollector().CollectFinish(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) + + return detectedEcsServices, nil + +} + +func (ec *EcsManager) describeServices(nextToken *string, EcsServices []*ecs.Service) ([]*ecs.Service, error) { + input := &ecs.ListClustersInput{ + MaxResults: nil, + NextToken: nextToken, + } + + resp, err := ec.client.ListClusters(input) + if err != nil { + log.WithField("error", err).Error("could not list any ecs clusters") + return nil, err + } + + if EcsServices == nil { + EcsServices = []*ecs.Service{} + } + + for _, clusterARN := range resp.ClusterArns { + var nextToeken *string = nil + for { + listServiceInput := &ecs.ListServicesInput{ + Cluster: clusterARN, + NextToken: nextToeken, + } + listServicesOutput, errr := ec.client.ListServices(listServiceInput) + if errr != nil { + log.WithField("error", errr).Error("could not list any services") + return nil, errr + } + describeServiceInput := &ecs.DescribeServicesInput{ + Cluster: clusterARN, + Include: nil, + Services: listServicesOutput.ServiceArns, + } + describeServiceOutput, errrr := ec.client.DescribeServices(describeServiceInput) + if errrr != nil { + log.WithField("error", errrr).Error("could not describe any services") + return nil, errrr + } + + EcsServices = append(EcsServices, describeServiceOutput.Services...) + if listServicesOutput.NextToken == nil { + break + } + nextToeken = listServicesOutput.NextToken + } + + //ListService fuer jedes arn + //Gibt arn von den ersten 10 services + //Arn ruft describeServices auf. Braucht arn und services array + //List of services + } + + if resp.NextToken != nil { + return ec.describeServices(resp.NextToken, EcsServices) + } + + return EcsServices, nil + +} From 67c7481319e567c56dea09aadd6a17e040ef065b Mon Sep 17 00:00:00 2001 From: DerOtt Date: Wed, 7 Jul 2021 12:40:09 +0200 Subject: [PATCH 46/68] Fixed a small bug --- collector/aws/resources/ecs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/aws/resources/ecs.go b/collector/aws/resources/ecs.go index c2896555..edaea7a0 100644 --- a/collector/aws/resources/ecs.go +++ b/collector/aws/resources/ecs.go @@ -90,7 +90,7 @@ func (ec *EcsManager) Detect(metrics []config.MetricConfig) (interface{}, error) period := int64(metric.Period.Seconds()) metricEndTime := now.Add(time.Duration(-metric.StartTime)) - tmp := strings.Split(*service.ServiceArn, "/") + tmp := strings.Split(*service.ClusterArn, "/") metricInput := awsCloudwatch.GetMetricStatisticsInput{ Dimensions: []*awsCloudwatch.Dimension{ { From dc5abf05080d1e3b83d0b14cc9496728d1331e45 Mon Sep 17 00:00:00 2001 From: DerOtt Date: Wed, 7 Jul 2021 14:33:49 +0200 Subject: [PATCH 47/68] Added more funcs lol. Price might be added --- collector/aws/resources/ecs.go | 2 +- collector/aws/resources/ecs_test.go | 263 ++++++++++++++++++++++++++++ 2 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 collector/aws/resources/ecs_test.go diff --git a/collector/aws/resources/ecs.go b/collector/aws/resources/ecs.go index edaea7a0..dbd2bd54 100644 --- a/collector/aws/resources/ecs.go +++ b/collector/aws/resources/ecs.go @@ -135,7 +135,7 @@ func (ec *EcsManager) Detect(metrics []config.MetricConfig) (interface{}, error) "service_name": *service.ServiceName, "region": ec.awsManager.GetRegion(), "launch_type": service.LaunchType, - }).Info("Redshift cluster detected as unutilized resource") + }).Info("ECS service detected as unutilized resource") tagsData := map[string]string{} if err == nil { diff --git a/collector/aws/resources/ecs_test.go b/collector/aws/resources/ecs_test.go new file mode 100644 index 00000000..441476d4 --- /dev/null +++ b/collector/aws/resources/ecs_test.go @@ -0,0 +1,263 @@ +package resources + +import ( + "errors" + awsTestutils "finala/collector/aws/testutils" + "finala/collector/config" + "finala/collector/testutils" + collectorTestutils "finala/collector/testutils" + awsClient "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ecs" + "reflect" + "testing" + "time" +) + +var defaultEcsDescribeServicesMock = ecs.DescribeServicesOutput{ + Services: []*ecs.Service{ + { + ServiceName: awsClient.String("MockService"), + ClusterArn: awsClient.String("arn:aws:ecs:us-west-2:1234567890:cluster/default"), + LaunchType: awsClient.String("EC2"), + CreatedAt: testutils.TimePointer(time.Now()), + ServiceArn: awsClient.String("arn:aws:ecs:us-west-2:1234567890:cluster/default/testECS"), + }, + { + ServiceName: awsClient.String("MockServiceFG"), + ClusterArn: awsClient.String("arn:aws:ecs:us-west-2:1234567890:cluster/default"), + LaunchType: awsClient.String("FARGATE"), + CreatedAt: testutils.TimePointer(time.Now()), + ServiceArn: awsClient.String("arn:aws:ecs:us-west-2:1234567891:cluster/default/testFG"), + }, + { + ServiceName: awsClient.String("MockServiceEX"), + ClusterArn: awsClient.String("arn:aws:ecs:us-west-2:1234567890:cluster/default"), + LaunchType: awsClient.String("EXTERNAL"), + CreatedAt: testutils.TimePointer(time.Now()), + ServiceArn: awsClient.String("arn:aws:ecs:us-west-2:1234567892:cluster/default/testEXTERNAL"), + }, + }, +} + +var defaultEcsListClustersMock = ecs.ListClustersOutput{ + ClusterArns: []*string{ + awsClient.String("arn:aws:ecs:us-west-2:1234567890:cluster/default"), + }, + NextToken: nil, +} + +var defaultEcsListServicesMock = ecs.ListServicesOutput{ + NextToken: nil, + ServiceArns: []*string{ + awsClient.String("arn:aws:ecs:us-west-2:1234567890:cluster/default/testECS"), + awsClient.String("arn:aws:ecs:us-west-2:1234567891:cluster/default/testFG"), + awsClient.String("arn:aws:ecs:us-west-2:1234567892:cluster/default/testEXTERNAL"), + }, +} + +type MockEcsClient struct { + responseDescribeServices ecs.DescribeServicesOutput + responseListClusters ecs.ListClustersOutput + responseListServices ecs.ListServicesOutput + err error +} + +func (ec *MockEcsClient) DescribeServices(input *ecs.DescribeServicesInput) (*ecs.DescribeServicesOutput, error) { + return &ec.responseDescribeServices, ec.err +} + +func (ec *MockEcsClient) ListClusters(input *ecs.ListClustersInput) (*ecs.ListClustersOutput, error) { + return &ec.responseListClusters, ec.err +} + +func (ec *MockEcsClient) ListServices(input *ecs.ListServicesInput) (*ecs.ListServicesOutput, error) { + return &ec.responseListServices, ec.err +} + +//detect und describeServices testen + +func TestEcsDescribeServices(t *testing.T) { + collector := collectorTestutils.NewMockCollector() + detector := awsTestutils.AWSManager(collector, nil, nil, "us-east-1") + + t.Run("valid", func(t *testing.T) { + mockClient := MockEcsClient{ + responseDescribeServices: defaultEcsDescribeServicesMock, + responseListClusters: defaultEcsListClustersMock, + responseListServices: defaultEcsListServicesMock, + err: nil, + } + + ecsInterface, err := NewEcsManager(detector, &mockClient) + if err != nil { + t.Fatalf("unexpected ecs error happened, got %v expected %v", err, nil) + } + + ecsManager, ok := ecsInterface.(*EcsManager) + if !ok { + t.Fatalf("unexpected ecs struct, got %s expected %s", reflect.TypeOf(ecsInterface), "*EcsManager") + } + + result, _ := ecsManager.describeServices(nil, nil) + + if len(result) != len(defaultEcsDescribeServicesMock.Services) { + t.Fatalf("unexpected ecs services count, got %d expected %d", len(result), len(defaultEcsDescribeServicesMock.Services)) + } + }) + + t.Run("error", func(t *testing.T) { + mockClient := MockEcsClient{ + responseDescribeServices: defaultEcsDescribeServicesMock, + responseListClusters: defaultEcsListClustersMock, + responseListServices: defaultEcsListServicesMock, + err: errors.New("error"), + } + + ecsInterface, err := NewEcsManager(detector, &mockClient) + + if err != nil { + t.Fatalf("unexpected ecs error happened, got %v expected %v", err, nil) + } + + ecsManager, ok := ecsInterface.(*EcsManager) + if !ok { + t.Fatalf("unexpected ecs struct, got %s expected %s", reflect.TypeOf(ecsInterface), "*EcsManager") + } + + _, err = ecsManager.describeServices(nil, nil) + + if err == nil { + t.Fatalf("unexpected describe services error, returned empty answer") + } + + }) + +} + +func TestEcsDetect(t *testing.T) { + //events 2 + //len 3 + + metricConfig := []config.MetricConfig{ + { + Description: "test description write capacity", + Data: []config.MetricDataConfiguration{ + { + Name: "TestMetric", + Statistic: "Sum", + }, + }, + Constraint: config.MetricConstraintConfig{ + Operator: "==", + Value: 5, + }, + Period: 1, + StartTime: 1, + }, + } + + collector := collectorTestutils.NewMockCollector() + mockCloudwatch := awsTestutils.NewMockCloudwatch(nil) + mockPrice := awsTestutils.NewMockPricing(nil) + detector := awsTestutils.AWSManager(collector, mockCloudwatch, mockPrice, "us-east-1") + + mockClient := MockEcsClient{ + responseDescribeServices: defaultEcsDescribeServicesMock, + responseListClusters: defaultEcsListClustersMock, + responseListServices: defaultEcsListServicesMock, + err: nil, + } + + ecsInterface, err := NewEcsManager(detector, &mockClient) + + if err != nil { + t.Fatalf("unexpected ecs error happened, got %v expected %v", err, nil) + } + + ecsManager, ok := ecsInterface.(*EcsManager) + if !ok { + t.Fatalf("unexpected ecs struct, got %s expected %s", reflect.TypeOf(ecsInterface), "*EcsManager") + } + + response, _ := ecsManager.Detect(metricConfig) + ecsResponse, ok := response.([]DetectedEcs) + + if !ok { + t.Fatalf("unexpected ecs struct, got %s expected %s", reflect.TypeOf(response), "[]DetectedEcs") + } + + if len(ecsResponse) != 3 { + t.Fatalf("unexpected ecs services detected, got %d expected %d", len(ecsResponse), 3) + } + + if len(collector.Events) != 3 { + t.Fatalf("unexpected collector ecs resources, got %d expected %d", len(collector.Events), 3) + } + + if len(collector.EventsCollectionStatus) != 2 { + t.Fatalf("unexpected resource status events count, got %d expected %d", len(collector.EventsCollectionStatus), 2) + } +} + +func TestEcsDetectError(t *testing.T) { + metricConfig := []config.MetricConfig{ + { + Description: "test description write capacity", + Data: []config.MetricDataConfiguration{ + { + Name: "TestMetric", + Statistic: "Sum", + }, + }, + Constraint: config.MetricConstraintConfig{ + Operator: "==", + Value: 5, + }, + Period: 1, + StartTime: 1, + }, + } + + collector := collectorTestutils.NewMockCollector() + mockCloudwatch := awsTestutils.NewMockCloudwatch(nil) + mockPrice := awsTestutils.NewMockPricing(nil) + detector := awsTestutils.AWSManager(collector, mockCloudwatch, mockPrice, "us-east-1") + + mockClient := MockEcsClient{ + responseDescribeServices: defaultEcsDescribeServicesMock, + responseListClusters: defaultEcsListClustersMock, + responseListServices: defaultEcsListServicesMock, + err: errors.New("error"), + } + + ecsInterface, err := NewEcsManager(detector, &mockClient) + + if err != nil { + t.Fatalf("unexpected ecs error happened, got %v expected %v", err, nil) + } + + ecsManager, ok := ecsInterface.(*EcsManager) + if !ok { + t.Fatalf("unexpected ecs struct, got %s expected %s", reflect.TypeOf(ecsInterface), "*EcsManager") + } + + response, _ := ecsManager.Detect(metricConfig) + ecsResponse, ok := response.([]DetectedEcs) + + if !ok { + t.Fatalf("unexpected ecs struct, got %s expected %s", reflect.TypeOf(response), "[]DetectedEcs") + } + + if len(ecsResponse) != 0 { + t.Fatalf("unexpected ecs services detected, got %d expected %d", len(ecsResponse), 3) + } + + if len(collector.Events) != 0 { + t.Fatalf("unexpected collector ecs resources, got %d expected %d", len(collector.Events), 1) + } + + if len(collector.EventsCollectionStatus) != 2 { + t.Fatalf("unexpected resource status events count, got %d expected %d", len(collector.EventsCollectionStatus), 2) + } + +} From f8677e8b3261820b5266b0bb368c13cc5007ddc4 Mon Sep 17 00:00:00 2001 From: daniebrill Date: Fri, 9 Jul 2021 23:03:29 +0200 Subject: [PATCH 48/68] Add account section for authentication to api.yaml + extend api config struct --- api/config/config.go | 15 ++++++++++++--- api/config/testutil/mock/config.yaml | 6 +++++- configuration/api.yaml | 6 +++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/api/config/config.go b/api/config/config.go index b63c28ec..38747a74 100644 --- a/api/config/config.go +++ b/api/config/config.go @@ -22,10 +22,20 @@ type StorageConfig struct { ElasticSearch ElasticsearchConfig `yaml:"elasticsearch"` } +type AccountConfig struct { + Name string `yaml:"name"` + Password string `yaml:"password"` +} + +type AuthenticationConfig struct { + Accounts []AccountConfig `yaml:"accounts"` +} + // APIConfig present the application config type APIConfig struct { - LogLevel string `yaml:"log_level"` - Storage StorageConfig `yaml:"storage"` + LogLevel string `yaml:"log_level"` + Storage StorageConfig `yaml:"storage"` + Authentication AuthenticationConfig `yaml:"authentication"` } // LoadAPI will load yaml file go struct @@ -49,6 +59,5 @@ func LoadAPI(location string) (APIConfig, error) { }).Info("override storage endpoint") config.Storage.ElasticSearch.Endpoints = strings.Split(overrideStorageEndpoint, ",") } - return config, nil } diff --git a/api/config/testutil/mock/config.yaml b/api/config/testutil/mock/config.yaml index f26c17b6..2a56fe14 100644 --- a/api/config/testutil/mock/config.yaml +++ b/api/config/testutil/mock/config.yaml @@ -7,4 +7,8 @@ storage: username: "" password: "" endpoints: - - http://127.0.0.1:9200 \ No newline at end of file + - http://127.0.0.1:9200 +authentication: + accounts: + - name: User + password: Password \ No newline at end of file diff --git a/configuration/api.yaml b/configuration/api.yaml index 82477c95..e9799596 100644 --- a/configuration/api.yaml +++ b/configuration/api.yaml @@ -6,4 +6,8 @@ storage: username: "" password: "" endpoints: - - http://127.0.0.1:9200 \ No newline at end of file + - http://127.0.0.1:9200 +authentication: + accounts: + - name: User + password: Finala \ No newline at end of file From 18d6bb9627179ba569f677c4c409bfefc3135fbf Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Sat, 10 Jul 2021 10:46:30 +0200 Subject: [PATCH 49/68] # SOS-30 in ecs get tasks if launchtype == Fargate --- collector/aws/resources/ecs.go | 63 ++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/collector/aws/resources/ecs.go b/collector/aws/resources/ecs.go index dbd2bd54..923bf46b 100644 --- a/collector/aws/resources/ecs.go +++ b/collector/aws/resources/ecs.go @@ -19,6 +19,8 @@ type EcsClientDescriptor interface { DescribeServices(input *ecs.DescribeServicesInput) (*ecs.DescribeServicesOutput, error) ListClusters(input *ecs.ListClustersInput) (*ecs.ListClustersOutput, error) ListServices(input *ecs.ListServicesInput) (*ecs.ListServicesOutput, error) + ListTasks(input *ecs.ListTasksInput) (*ecs.ListTasksOutput, error) + DescribeTasks(input *ecs.DescribeTasksInput) (*ecs.DescribeTasksOutput, error) } type EcsManager struct { @@ -144,14 +146,31 @@ func (ec *EcsManager) Detect(metrics []config.MetricConfig) (interface{}, error) } } + pricePerHour := float64(0) + + if *service.LaunchType == ecs.LaunchTypeFargate { + Tasks, err := ec.describeTasks(service.ClusterArn, service.ServiceName, nil, nil) + if err == nil { + for _, task := range Tasks { + log.WithFields(log.Fields{ + "arn": *task.TaskArn, + "cpu": *task.Cpu, + "memory": *task.Memory, + }).Info("task") + } + } + } + ecss := DetectedEcs{ Region: ec.awsManager.GetRegion(), Metric: metric.Description, LaunchType: *service.LaunchType, PriceDetectedFields: collector.PriceDetectedFields{ - ResourceID: *service.ServiceArn, - LaunchTime: *service.CreatedAt, - Tag: tagsData, + ResourceID: *service.ServiceArn, + LaunchTime: *service.CreatedAt, + PricePerHour: pricePerHour, + PricePerMonth: pricePerHour * collector.TotalMonthHours, + Tag: tagsData, }, AccountSpecifiedFields: collector.AccountSpecifiedFields{ AccountID: *ec.awsManager.GetAccountIdentity().Account, @@ -238,3 +257,41 @@ func (ec *EcsManager) describeServices(nextToken *string, EcsServices []*ecs.Ser return EcsServices, nil } + +func (ec *EcsManager) describeTasks(clusterArn *string, serviceName *string, nextToken *string, Tasks []*ecs.Task) ([]*ecs.Task, error) { + + listInput := &ecs.ListTasksInput{ + Cluster: clusterArn, + NextToken: nextToken, + ServiceName: serviceName, + } + + listResp, err := ec.client.ListTasks(listInput) + if err != nil { + log.WithField("error", err).Error("could not list any ecs task") + return nil, err + } + + if Tasks == nil { + Tasks = []*ecs.Task{} + } + + describeInput := &ecs.DescribeTasksInput{ + Cluster: clusterArn, + Tasks: listResp.TaskArns, + } + + describeResp, err := ec.client.DescribeTasks(describeInput) + if err != nil { + log.WithField("error", err).Error("could not describe any ecs task") + return nil, err + } + + Tasks = append(Tasks, describeResp.Tasks...) + + if listResp.NextToken != nil { + return ec.describeTasks(clusterArn, serviceName, listResp.NextToken, Tasks) + } + + return Tasks, nil +} From e88c686e32b719e87b732c117403bd2d21ebdd1f Mon Sep 17 00:00:00 2001 From: DerOtt Date: Sat, 10 Jul 2021 09:56:09 +0200 Subject: [PATCH 50/68] Cleanup and more --- collector/aws/resources/ecs.go | 83 +++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/collector/aws/resources/ecs.go b/collector/aws/resources/ecs.go index 923bf46b..bdf65081 100644 --- a/collector/aws/resources/ecs.go +++ b/collector/aws/resources/ecs.go @@ -7,10 +7,13 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "fmt" awsClient "github.com/aws/aws-sdk-go/aws" awsCloudwatch "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/ecs" + "github.com/aws/aws-sdk-go/service/pricing" log "github.com/sirupsen/logrus" + "strconv" "strings" "time" ) @@ -24,10 +27,11 @@ type EcsClientDescriptor interface { } type EcsManager struct { - client EcsClientDescriptor - awsManager common.AWSManager - namespace string - Name collector.ResourceIdentifier + client EcsClientDescriptor + awsManager common.AWSManager + namespace string + Name collector.ResourceIdentifier + servicePricingCode string } type DetectedEcs struct { @@ -53,10 +57,11 @@ func NewEcsManager(awsManager common.AWSManager, client interface{}) (common.Res } return &EcsManager{ - client: ecsClient, - awsManager: awsManager, - namespace: "AWS/ECS", - Name: awsManager.GetResourceIdentifier("ECS"), + client: ecsClient, + awsManager: awsManager, + namespace: "AWS/ECS", + Name: awsManager.GetResourceIdentifier("ECS"), + servicePricingCode: "AmazonECS", }, nil } @@ -151,12 +156,70 @@ func (ec *EcsManager) Detect(metrics []config.MetricConfig) (interface{}, error) if *service.LaunchType == ecs.LaunchTypeFargate { Tasks, err := ec.describeTasks(service.ClusterArn, service.ServiceName, nil, nil) if err == nil { + + pricingRegionPrefix, err := ec.awsManager.GetPricingClient().GetRegionPrefix(ec.awsManager.GetRegion()) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "region": ec.awsManager.GetRegion(), + }).Error("Could not get pricing region prefix") + ec.awsManager.GetCollector().CollectError(ec.Name, err) + return detectedEcsServices, err + } + for _, task := range Tasks { log.WithFields(log.Fields{ "arn": *task.TaskArn, "cpu": *task.Cpu, "memory": *task.Memory, }).Info("task") + + priceGBPerHour, err := ec.awsManager.GetPricingClient().GetPrice(pricing.GetProductsInput{ServiceCode: &ec.servicePricingCode, Filters: []*pricing.Filter{ + {Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("usageType"), + Value: awsClient.String(fmt.Sprintf("%sFargate-GB-Hours", pricingRegionPrefix)), + }, + }}, "", ec.awsManager.GetRegion()) + + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "region": ec.awsManager.GetRegion(), + }).Error("could not get ECS per GB price") + ec.awsManager.GetCollector().CollectError(ec.Name, err) + return detectedEcsServices, err + } + multiplier, err := strconv.ParseFloat(*task.Memory, 64) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "Memory": *task.Memory, + }).Error("Cast error in memory cost calc") + } else { + pricePerHour += multiplier / 1024 * priceGBPerHour + } + + priceCPUPerHour, err := ec.awsManager.GetPricingClient().GetPrice(pricing.GetProductsInput{ServiceCode: &ec.servicePricingCode, Filters: []*pricing.Filter{ + {Type: awsClient.String("TERM_MATCH"), + Field: awsClient.String("usageType"), + Value: awsClient.String(fmt.Sprintf("%sFargate-vCPU-Hours:perCPU", pricingRegionPrefix)), + }, + }}, "", ec.awsManager.GetRegion()) + + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "region": ec.awsManager.GetRegion(), + }).Error("could not get ECS per CPU price") + ec.awsManager.GetCollector().CollectError(ec.Name, err) + return detectedEcsServices, err + } + + multiplier, err = strconv.ParseFloat(*task.Cpu, 64) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "CPU": *task.Cpu, + }).Error("Cast error in cpu cost calc") + } else { + pricePerHour += multiplier / 1024 * priceCPUPerHour + } + } } } @@ -244,10 +307,6 @@ func (ec *EcsManager) describeServices(nextToken *string, EcsServices []*ecs.Ser nextToeken = listServicesOutput.NextToken } - //ListService fuer jedes arn - //Gibt arn von den ersten 10 services - //Arn ruft describeServices auf. Braucht arn und services array - //List of services } if resp.NextToken != nil { From 1161a8becaf0a0db49b4fa5fb554c3604fa967d7 Mon Sep 17 00:00:00 2001 From: DerOtt Date: Sat, 10 Jul 2021 10:19:45 +0200 Subject: [PATCH 51/68] Did more things --- collector/aws/resources/ecs.go | 11 +++----- collector/aws/resources/ecs_test.go | 39 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/collector/aws/resources/ecs.go b/collector/aws/resources/ecs.go index bdf65081..b43fa28d 100644 --- a/collector/aws/resources/ecs.go +++ b/collector/aws/resources/ecs.go @@ -163,15 +163,10 @@ func (ec *EcsManager) Detect(metrics []config.MetricConfig) (interface{}, error) "region": ec.awsManager.GetRegion(), }).Error("Could not get pricing region prefix") ec.awsManager.GetCollector().CollectError(ec.Name, err) - return detectedEcsServices, err + } for _, task := range Tasks { - log.WithFields(log.Fields{ - "arn": *task.TaskArn, - "cpu": *task.Cpu, - "memory": *task.Memory, - }).Info("task") priceGBPerHour, err := ec.awsManager.GetPricingClient().GetPrice(pricing.GetProductsInput{ServiceCode: &ec.servicePricingCode, Filters: []*pricing.Filter{ {Type: awsClient.String("TERM_MATCH"), @@ -185,7 +180,7 @@ func (ec *EcsManager) Detect(metrics []config.MetricConfig) (interface{}, error) "region": ec.awsManager.GetRegion(), }).Error("could not get ECS per GB price") ec.awsManager.GetCollector().CollectError(ec.Name, err) - return detectedEcsServices, err + continue } multiplier, err := strconv.ParseFloat(*task.Memory, 64) if err != nil { @@ -208,7 +203,7 @@ func (ec *EcsManager) Detect(metrics []config.MetricConfig) (interface{}, error) "region": ec.awsManager.GetRegion(), }).Error("could not get ECS per CPU price") ec.awsManager.GetCollector().CollectError(ec.Name, err) - return detectedEcsServices, err + continue } multiplier, err = strconv.ParseFloat(*task.Cpu, 64) diff --git a/collector/aws/resources/ecs_test.go b/collector/aws/resources/ecs_test.go index 441476d4..2ed26bb9 100644 --- a/collector/aws/resources/ecs_test.go +++ b/collector/aws/resources/ecs_test.go @@ -55,13 +55,44 @@ var defaultEcsListServicesMock = ecs.ListServicesOutput{ }, } +var defaultEcsListTasksMock = ecs.ListTasksOutput{ + NextToken: nil, + TaskArns: []*string{ + awsClient.String("arn:aws:ecs:us-west-2:1234567890:cluster/default/testTaskOne"), + }, +} + +var defaultEcsDescribeTasksMock = ecs.DescribeTasksOutput{ + Failures: nil, + Tasks: []*ecs.Task{ + { + Cpu: awsClient.String("1024"), + Memory: awsClient.String("1024"), + TaskArn: awsClient.String("arn:aws:ecs:us-west-2:1234567890:cluster/default/testTaskOne"), + ClusterArn: awsClient.String("arn:aws:ecs:us-west-2:1234567890:cluster/default/testFG"), + }, + }, +} + +//listtaks und describe tasks + type MockEcsClient struct { responseDescribeServices ecs.DescribeServicesOutput responseListClusters ecs.ListClustersOutput responseListServices ecs.ListServicesOutput + responseDescribeTasks ecs.DescribeTasksOutput + responseListTasks ecs.ListTasksOutput err error } +func (ec *MockEcsClient) ListTasks(input *ecs.ListTasksInput) (*ecs.ListTasksOutput, error) { + return &ec.responseListTasks, ec.err +} + +func (ec *MockEcsClient) DescribeTasks(input *ecs.DescribeTasksInput) (*ecs.DescribeTasksOutput, error) { + return &ec.responseDescribeTasks, ec.err +} + func (ec *MockEcsClient) DescribeServices(input *ecs.DescribeServicesInput) (*ecs.DescribeServicesOutput, error) { return &ec.responseDescribeServices, ec.err } @@ -85,6 +116,8 @@ func TestEcsDescribeServices(t *testing.T) { responseDescribeServices: defaultEcsDescribeServicesMock, responseListClusters: defaultEcsListClustersMock, responseListServices: defaultEcsListServicesMock, + responseDescribeTasks: defaultEcsDescribeTasksMock, + responseListTasks: defaultEcsListTasksMock, err: nil, } @@ -110,6 +143,8 @@ func TestEcsDescribeServices(t *testing.T) { responseDescribeServices: defaultEcsDescribeServicesMock, responseListClusters: defaultEcsListClustersMock, responseListServices: defaultEcsListServicesMock, + responseDescribeTasks: defaultEcsDescribeTasksMock, + responseListTasks: defaultEcsListTasksMock, err: errors.New("error"), } @@ -165,6 +200,8 @@ func TestEcsDetect(t *testing.T) { responseDescribeServices: defaultEcsDescribeServicesMock, responseListClusters: defaultEcsListClustersMock, responseListServices: defaultEcsListServicesMock, + responseDescribeTasks: defaultEcsDescribeTasksMock, + responseListTasks: defaultEcsListTasksMock, err: nil, } @@ -227,6 +264,8 @@ func TestEcsDetectError(t *testing.T) { responseDescribeServices: defaultEcsDescribeServicesMock, responseListClusters: defaultEcsListClustersMock, responseListServices: defaultEcsListServicesMock, + responseDescribeTasks: defaultEcsDescribeTasksMock, + responseListTasks: defaultEcsListTasksMock, err: errors.New("error"), } From 1c2ef4b4e9d01c5127267859846d9e3a428f9124 Mon Sep 17 00:00:00 2001 From: DerOtt Date: Mon, 12 Jul 2021 11:33:45 +0200 Subject: [PATCH 52/68] Removed some comments --- collector/aws/resources/ecs_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/collector/aws/resources/ecs_test.go b/collector/aws/resources/ecs_test.go index 2ed26bb9..92fd842f 100644 --- a/collector/aws/resources/ecs_test.go +++ b/collector/aws/resources/ecs_test.go @@ -74,8 +74,6 @@ var defaultEcsDescribeTasksMock = ecs.DescribeTasksOutput{ }, } -//listtaks und describe tasks - type MockEcsClient struct { responseDescribeServices ecs.DescribeServicesOutput responseListClusters ecs.ListClustersOutput @@ -105,8 +103,6 @@ func (ec *MockEcsClient) ListServices(input *ecs.ListServicesInput) (*ecs.ListSe return &ec.responseListServices, ec.err } -//detect und describeServices testen - func TestEcsDescribeServices(t *testing.T) { collector := collectorTestutils.NewMockCollector() detector := awsTestutils.AWSManager(collector, nil, nil, "us-east-1") @@ -170,8 +166,6 @@ func TestEcsDescribeServices(t *testing.T) { } func TestEcsDetect(t *testing.T) { - //events 2 - //len 3 metricConfig := []config.MetricConfig{ { From d9b9549192fb2c9918546eaa8af1aab650b8290f Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 14 Jul 2021 17:33:07 +0200 Subject: [PATCH 53/68] # SOS-48 add login api endpoint add login function in server --- api/route.go | 44 +++++++++++++++++++++++++++++++++++++++++ api/server.go | 20 +++++++++++-------- api/server_test.go | 4 +++- api/testutils/server.go | 12 +++++++++++ cmd/api.go | 2 +- go.mod | 1 + go.sum | 7 +------ 7 files changed, 74 insertions(+), 16 deletions(-) diff --git a/api/route.go b/api/route.go index 756c2f32..e2738cf6 100644 --- a/api/route.go +++ b/api/route.go @@ -4,6 +4,7 @@ import ( "encoding/json" "finala/api/httpparameters" "finala/api/storage" + "github.com/dgrijalva/jwt-go" "io/ioutil" "net/http" "net/url" @@ -181,6 +182,49 @@ func (server *Server) DetectEvents(resp http.ResponseWriter, req *http.Request) } +func (server *Server) Login(resp http.ResponseWriter, req *http.Request) { + buf, bodyErr := ioutil.ReadAll(req.Body) + + if bodyErr != nil { + server.JSONWrite(resp, http.StatusBadRequest, HttpErrorResponse{Error: bodyErr.Error()}) + return + } + + var detectUser map[string]string + err := json.Unmarshal(buf, &detectUser) + if err != nil { + server.JSONWrite(resp, http.StatusBadRequest, HttpErrorResponse{Error: err.Error()}) + return + } + + for _, user := range server.authentication.Accounts { + if detectUser["username"] == user.Name && detectUser["password"] == user.Password { + + expTime := time.Now().Add(time.Minute * 5) + + atClaims := jwt.MapClaims{} + atClaims["authorized"] = true + atClaims["user_id"] = user.Name + atClaims["exp"] = expTime.Unix() + at := jwt.NewWithClaims(jwt.SigningMethodHS256, atClaims) + token, err := at.SignedString([]byte("secret")) + if err != nil { + server.JSONWrite(resp, http.StatusBadRequest, HttpErrorResponse{Error: err.Error()}) + return + } + + cookie := http.Cookie{ + Name: "jwt", + Value: token, + Expires: expTime, + } + + http.SetCookie(resp, &cookie) + + } + } +} + //NotFoundRoute return when route not found func (server *Server) NotFoundRoute(resp http.ResponseWriter, req *http.Request) { server.JSONWrite(resp, http.StatusNotFound, HttpErrorResponse{Error: "Path not found"}) diff --git a/api/server.go b/api/server.go index 357c9c69..909058d3 100644 --- a/api/server.go +++ b/api/server.go @@ -3,6 +3,7 @@ package api import ( "context" "encoding/json" + "finala/api/config" "fmt" "net/http" "time" @@ -24,21 +25,23 @@ const ( // Server is the API server struct type Server struct { - router *mux.Router - httpserver *http.Server - storage storage.StorageDescriber - version version.VersionManagerDescriptor + router *mux.Router + httpserver *http.Server + storage storage.StorageDescriber + authentication config.AuthenticationConfig + version version.VersionManagerDescriptor } // NewServer returns a new Server -func NewServer(port int, storage storage.StorageDescriber, version version.VersionManagerDescriptor) *Server { +func NewServer(port int, storage storage.StorageDescriber, version version.VersionManagerDescriptor, auth config.AuthenticationConfig) *Server { router := mux.NewRouter() corsObj := handlers.AllowedOrigins([]string{"*"}) return &Server{ - router: router, - storage: storage, - version: version, + router: router, + storage: storage, + version: version, + authentication: auth, httpserver: &http.Server{ Handler: handlers.CORS(corsObj)(router), Addr: fmt.Sprintf("0.0.0.0:%d", port), @@ -86,6 +89,7 @@ func (server *Server) BindEndpoints() { server.router.HandleFunc("/api/v1/trends/{type}", server.GetResourceTrends).Methods("GET") server.router.HandleFunc("/api/v1/tags/{executionID}", server.GetExecutionTags).Methods("GET") server.router.HandleFunc("/api/v1/detect-events/{executionID}", server.DetectEvents).Methods("POST") + server.router.HandleFunc("/api/v1/login", server.Login).Methods("GET") server.router.HandleFunc("/api/v1/version", server.VersionHandler).Methods("GET") server.router.HandleFunc("/api/v1/health", server.HealthCheckHandler).Methods("GET") server.router.NotFoundHandler = http.HandlerFunc(server.NotFoundRoute) diff --git a/api/server_test.go b/api/server_test.go index 55192ec4..2bc33c9d 100644 --- a/api/server_test.go +++ b/api/server_test.go @@ -20,7 +20,9 @@ func MockServer() (*api.Server, *testutils.MockStorage) { version := testutils.NewMockVersion() mockStorage := testutils.NewMockStorage() - server := api.NewServer(9090, mockStorage, version) + mockAuthenticationConfig := testutils.GetAuthenticationConfig() + + server := api.NewServer(9090, mockStorage, version, mockAuthenticationConfig) return server, mockStorage } diff --git a/api/testutils/server.go b/api/testutils/server.go index 6bd0189a..f0c4c397 100644 --- a/api/testutils/server.go +++ b/api/testutils/server.go @@ -1,6 +1,7 @@ package testutils import ( + "finala/api/config" "net" "net/http" "strings" @@ -10,6 +11,17 @@ import ( "github.com/gorilla/mux" ) +func GetAuthenticationConfig() config.AuthenticationConfig { + return config.AuthenticationConfig{ + Accounts: []config.AccountConfig{ + { + Name: "User", + Password: "Finala", + }, + }, + } +} + type MockWebserver struct { Port string Router *mux.Router diff --git a/cmd/api.go b/cmd/api.go index 42430ec9..b8efc688 100644 --- a/cmd/api.go +++ b/cmd/api.go @@ -42,7 +42,7 @@ var apiServer = &cobra.Command{ os.Exit(1) } - apiManager := api.NewServer(port, storage, versionManager) + apiManager := api.NewServer(port, storage, versionManager, configStruct.Authentication) apiStopper := serverutil.RunAll(apiManager).StopFunc diff --git a/go.mod b/go.mod index 476ed36e..36a815bf 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/aws/aws-sdk-go v1.31.3 + github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dustin/go-humanize v1.0.0 github.com/gorilla/handlers v1.4.2 github.com/gorilla/mux v1.7.4 diff --git a/go.sum b/go.sum index 49880209..8fc491d2 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= @@ -71,7 +72,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -125,7 +125,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -141,7 +140,6 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -172,9 +170,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd h1:r7DufRZuZbWB7j439YfAzP8RPDa9unLkpwQKUYbIMPI= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f h1:68K/z8GLUxV76xGSqwTWw2gyk/jwn79LUL43rES2g8o= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -198,7 +194,6 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 3919a61dc60f574663f49d8da0efc5622a918672 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 14 Jul 2021 21:22:05 +0200 Subject: [PATCH 54/68] # SOS-48 add test func Error output corrected with correct status code --- api/route.go | 7 ++++--- api/server_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/api/route.go b/api/route.go index e2738cf6..76afb8c6 100644 --- a/api/route.go +++ b/api/route.go @@ -198,7 +198,7 @@ func (server *Server) Login(resp http.ResponseWriter, req *http.Request) { } for _, user := range server.authentication.Accounts { - if detectUser["username"] == user.Name && detectUser["password"] == user.Password { + if detectUser["Username"] == user.Name && detectUser["Password"] == user.Password { expTime := time.Now().Add(time.Minute * 5) @@ -209,7 +209,7 @@ func (server *Server) Login(resp http.ResponseWriter, req *http.Request) { at := jwt.NewWithClaims(jwt.SigningMethodHS256, atClaims) token, err := at.SignedString([]byte("secret")) if err != nil { - server.JSONWrite(resp, http.StatusBadRequest, HttpErrorResponse{Error: err.Error()}) + server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) return } @@ -220,9 +220,10 @@ func (server *Server) Login(resp http.ResponseWriter, req *http.Request) { } http.SetCookie(resp, &cookie) - + return } } + server.JSONWrite(resp, http.StatusUnauthorized, "{\"message\":\"Login data not authorized\"}") } //NotFoundRoute return when route not found diff --git a/api/server_test.go b/api/server_test.go index 2bc33c9d..06fce1cf 100644 --- a/api/server_test.go +++ b/api/server_test.go @@ -511,3 +511,51 @@ func TestVersion(t *testing.T) { } } + +func TestLogin(t *testing.T) { + ms, _ := MockServer() + ms.BindEndpoints() + ms.Serve() + + type testAccount struct { + Username string + Password string + } + + testCases := []struct { + endpoint string + expectedStatusCode int + BodyRequest testAccount + }{ + {"/api/v1/login", http.StatusOK, testAccount{Username: "User", Password: "Finala"}}, + {"/api/v1/login", http.StatusUnauthorized, testAccount{Username: "", Password: ""}}, + } + + for _, test := range testCases { + t.Run(test.endpoint, func(t *testing.T) { + rr := httptest.NewRecorder() + buf, err := json.Marshal(test.BodyRequest) + if err != nil { + t.Fatal(err) + } + + req, err := http.NewRequest("GET", test.endpoint, bytes.NewBuffer(buf)) + if err != nil { + t.Fatal(err) + } + + ms.Router().ServeHTTP(rr, req) + if rr.Code != test.expectedStatusCode { + t.Fatalf("handler returned wrong status code: got %v want %v", rr.Code, test.expectedStatusCode) + } + + if test.expectedStatusCode == http.StatusOK { + if cookie := rr.Header().Get("Set-Cookie"); cookie == "" { + t.Fatalf("unexpected error, got %s expected jwt={SomeValue}", cookie) + } + } + + }) + } + +} From 0f438d4bcf1887b07e3da0dd30252c24c85bded7 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 14 Jul 2021 22:24:18 +0200 Subject: [PATCH 55/68] token expire time from 5 minute to 1 hour --- api/route.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/route.go b/api/route.go index 76afb8c6..eab2dfc2 100644 --- a/api/route.go +++ b/api/route.go @@ -200,7 +200,7 @@ func (server *Server) Login(resp http.ResponseWriter, req *http.Request) { for _, user := range server.authentication.Accounts { if detectUser["Username"] == user.Name && detectUser["Password"] == user.Password { - expTime := time.Now().Add(time.Minute * 5) + expTime := time.Now().Add(time.Hour * 1) atClaims := jwt.MapClaims{} atClaims["authorized"] = true From 7c316282e7cb783e64c0f1db49f9a4316cd6403e Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Mon, 19 Jul 2021 14:18:30 +0200 Subject: [PATCH 56/68] add authentication with cookie change tests or authentication with cookie --- api/route.go | 24 +++++++++++++++++++++++- api/server.go | 12 ++++++------ api/server_test.go | 6 ++++++ api/testutils/server.go | 24 ++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/api/route.go b/api/route.go index 45a78e2f..6bd7682f 100644 --- a/api/route.go +++ b/api/route.go @@ -4,8 +4,8 @@ import ( "encoding/json" "finala/api/httpparameters" "finala/api/storage" - "github.com/dgrijalva/jwt-go" "fmt" + "github.com/dgrijalva/jwt-go" "io/ioutil" "net/http" "net/url" @@ -247,6 +247,28 @@ func (server *Server) VersionHandler(resp http.ResponseWriter, req *http.Request server.JSONWrite(resp, http.StatusOK, version) } +func (server *Server) middleware(next http.Handler) http.HandlerFunc { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + cookie, err := req.Cookie("jwt") + log.Println(cookie) + if err != nil { + server.JSONWrite(resp, http.StatusUnauthorized, HttpErrorResponse{Error: "Authorize cookie not found"}) + return + } + claims := jwt.MapClaims{} + _, err = jwt.ParseWithClaims(cookie.Value, claims, func(token *jwt.Token) (interface{}, error) { + return []byte("secret"), nil + }) + + if err != nil { + server.JSONWrite(resp, http.StatusUnauthorized, HttpErrorResponse{Error: err.Error()}) + return + } + + next.ServeHTTP(resp, req) + }) +} + //Returns json thingy wingy dingy i dont know how func (server *Server) GetReport(resp http.ResponseWriter, req *http.Request) { queryParams := req.URL.Query() diff --git a/api/server.go b/api/server.go index 7018863e..0557ae3a 100644 --- a/api/server.go +++ b/api/server.go @@ -82,12 +82,12 @@ func (server *Server) Serve() serverutil.StopFunc { // BindEndpoints sets up the router to handle API endpoints func (server *Server) BindEndpoints() { - server.router.HandleFunc("/api/v1/summary/{executionID}", server.GetSummary).Methods("GET") - server.router.HandleFunc("/api/v1/executions", server.GetExecutions).Methods("GET") - server.router.HandleFunc("/api/v1/accounts/{executionID}", server.GetAccounts).Methods(("GET")) - server.router.HandleFunc("/api/v1/resources/{type}", server.GetResourceData).Methods("GET") - server.router.HandleFunc("/api/v1/trends/{type}", server.GetResourceTrends).Methods("GET") - server.router.HandleFunc("/api/v1/tags/{executionID}", server.GetExecutionTags).Methods("GET") + server.router.HandleFunc("/api/v1/summary/{executionID}", server.middleware(http.HandlerFunc(server.GetSummary))).Methods("GET") + server.router.HandleFunc("/api/v1/executions", server.middleware(http.HandlerFunc(server.GetExecutions))).Methods("GET") + server.router.HandleFunc("/api/v1/accounts/{executionID}", server.middleware(http.HandlerFunc(server.GetAccounts))).Methods(("GET")) + server.router.HandleFunc("/api/v1/resources/{type}", server.middleware(http.HandlerFunc(server.GetResourceData))).Methods("GET") + server.router.HandleFunc("/api/v1/trends/{type}", server.middleware(http.HandlerFunc(server.GetResourceTrends))).Methods("GET") + server.router.HandleFunc("/api/v1/tags/{executionID}", server.middleware(http.HandlerFunc(server.GetExecutionTags))).Methods("GET") server.router.HandleFunc("/api/v1/detect-events/{executionID}", server.DetectEvents).Methods("POST") server.router.HandleFunc("/api/v1/login", server.Login).Methods("GET") server.router.HandleFunc("/api/v1/version", server.VersionHandler).Methods("GET") diff --git a/api/server_test.go b/api/server_test.go index 06fce1cf..c0f9b8b1 100644 --- a/api/server_test.go +++ b/api/server_test.go @@ -105,6 +105,7 @@ func TestGetSummary(t *testing.T) { rr := httptest.NewRecorder() req, err := http.NewRequest("GET", test.endpoint, nil) + req.AddCookie(testutils.GetTestCookie()) if err != nil { t.Fatal(err) } @@ -158,6 +159,7 @@ func TestGetResourcesData(t *testing.T) { t.Run(test.endpoint, func(t *testing.T) { rr := httptest.NewRecorder() req, err := http.NewRequest("GET", test.endpoint, nil) + req.AddCookie(testutils.GetTestCookie()) if err != nil { t.Fatal(err) } @@ -211,6 +213,7 @@ func TestGetExecutions(t *testing.T) { t.Run(test.endpoint, func(t *testing.T) { rr := httptest.NewRecorder() req, err := http.NewRequest("GET", test.endpoint, nil) + req.AddCookie(testutils.GetTestCookie()) if err != nil { t.Fatal(err) } @@ -259,6 +262,7 @@ func TestGetAccounts(t *testing.T) { rr := httptest.NewRecorder() req, err := http.NewRequest("GET", test.endpoint, nil) + req.AddCookie(testutils.GetTestCookie()) if err != nil { t.Fatal(err) } @@ -362,6 +366,7 @@ func TestGetExecutionTags(t *testing.T) { t.Run(test.endpoint, func(t *testing.T) { rr := httptest.NewRecorder() req, err := http.NewRequest("GET", test.endpoint, nil) + req.AddCookie(testutils.GetTestCookie()) if err != nil { t.Fatal(err) } @@ -416,6 +421,7 @@ func TestGetResourceTrends(t *testing.T) { t.Run(test.endpoint, func(t *testing.T) { rr := httptest.NewRecorder() req, err := http.NewRequest("GET", test.endpoint, nil) + req.AddCookie(testutils.GetTestCookie()) if err != nil { t.Fatal(err) } diff --git a/api/testutils/server.go b/api/testutils/server.go index f0c4c397..a299005e 100644 --- a/api/testutils/server.go +++ b/api/testutils/server.go @@ -2,9 +2,11 @@ package testutils import ( "finala/api/config" + "github.com/dgrijalva/jwt-go" "net" "net/http" "strings" + "time" log "github.com/sirupsen/logrus" @@ -22,6 +24,28 @@ func GetAuthenticationConfig() config.AuthenticationConfig { } } +func GetTestCookie() *http.Cookie { + + expTime := time.Now().Add(time.Minute * 5) + + atClaims := jwt.MapClaims{} + atClaims["authorized"] = true + atClaims["user_id"] = "TestCookie" + atClaims["exp"] = expTime.Unix() + at := jwt.NewWithClaims(jwt.SigningMethodHS256, atClaims) + token, err := at.SignedString([]byte("secret")) + if err != nil { + return nil + } + + cookie := http.Cookie{ + Name: "jwt", + Value: token, + Expires: expTime, + } + return &cookie +} + type MockWebserver struct { Port string Router *mux.Router From 7071ebd8cefe2f38934f0fc89c113ba61caac9cb Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Mon, 19 Jul 2021 14:49:00 +0200 Subject: [PATCH 57/68] add authentication throttle --- api/config/config.go | 1 + api/config/testutil/mock/config.yaml | 1 + api/route.go | 85 ++++++++++++++++++---------- api/testutils/server.go | 1 + configuration/api.yaml | 1 + 5 files changed, 58 insertions(+), 31 deletions(-) diff --git a/api/config/config.go b/api/config/config.go index 38747a74..5200bfe7 100644 --- a/api/config/config.go +++ b/api/config/config.go @@ -28,6 +28,7 @@ type AccountConfig struct { } type AuthenticationConfig struct { + Enabled bool `yaml:"enabled"` Accounts []AccountConfig `yaml:"accounts"` } diff --git a/api/config/testutil/mock/config.yaml b/api/config/testutil/mock/config.yaml index 2a56fe14..af95054a 100644 --- a/api/config/testutil/mock/config.yaml +++ b/api/config/testutil/mock/config.yaml @@ -9,6 +9,7 @@ storage: endpoints: - http://127.0.0.1:9200 authentication: + enabled: true accounts: - name: User password: Password \ No newline at end of file diff --git a/api/route.go b/api/route.go index 6bd7682f..7d4096ba 100644 --- a/api/route.go +++ b/api/route.go @@ -184,47 +184,70 @@ func (server *Server) DetectEvents(resp http.ResponseWriter, req *http.Request) } func (server *Server) Login(resp http.ResponseWriter, req *http.Request) { - buf, bodyErr := ioutil.ReadAll(req.Body) + if server.authentication.Enabled { + buf, bodyErr := ioutil.ReadAll(req.Body) - if bodyErr != nil { - server.JSONWrite(resp, http.StatusBadRequest, HttpErrorResponse{Error: bodyErr.Error()}) - return - } + if bodyErr != nil { + server.JSONWrite(resp, http.StatusBadRequest, HttpErrorResponse{Error: bodyErr.Error()}) + return + } - var detectUser map[string]string - err := json.Unmarshal(buf, &detectUser) - if err != nil { - server.JSONWrite(resp, http.StatusBadRequest, HttpErrorResponse{Error: err.Error()}) - return - } + var detectUser map[string]string + err := json.Unmarshal(buf, &detectUser) + if err != nil { + server.JSONWrite(resp, http.StatusBadRequest, HttpErrorResponse{Error: err.Error()}) + return + } - for _, user := range server.authentication.Accounts { - if detectUser["Username"] == user.Name && detectUser["Password"] == user.Password { + for _, user := range server.authentication.Accounts { + if detectUser["Username"] == user.Name && detectUser["Password"] == user.Password { - expTime := time.Now().Add(time.Hour * 1) + expTime := time.Now().Add(time.Hour * 1) - atClaims := jwt.MapClaims{} - atClaims["authorized"] = true - atClaims["user_id"] = user.Name - atClaims["exp"] = expTime.Unix() - at := jwt.NewWithClaims(jwt.SigningMethodHS256, atClaims) - token, err := at.SignedString([]byte("secret")) - if err != nil { - server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) - return - } + atClaims := jwt.MapClaims{} + atClaims["authorized"] = true + atClaims["user_id"] = user.Name + atClaims["exp"] = expTime.Unix() + at := jwt.NewWithClaims(jwt.SigningMethodHS256, atClaims) + token, err := at.SignedString([]byte("secret")) + if err != nil { + server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) + return + } - cookie := http.Cookie{ - Name: "jwt", - Value: token, - Expires: expTime, - } + cookie := http.Cookie{ + Name: "jwt", + Value: token, + Expires: expTime, + } - http.SetCookie(resp, &cookie) + http.SetCookie(resp, &cookie) + return + } + } + server.JSONWrite(resp, http.StatusUnauthorized, "{\"message\":\"Login data not authorized\"}") + } else { + expTime := time.Now().Add(time.Hour * 1) + + atClaims := jwt.MapClaims{} + atClaims["authorized"] = true + atClaims["user_id"] = "user" + atClaims["exp"] = expTime.Unix() + at := jwt.NewWithClaims(jwt.SigningMethodHS256, atClaims) + token, err := at.SignedString([]byte("secret")) + if err != nil { + server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) return } + + cookie := http.Cookie{ + Name: "jwt", + Value: token, + Expires: expTime, + } + + http.SetCookie(resp, &cookie) } - server.JSONWrite(resp, http.StatusUnauthorized, "{\"message\":\"Login data not authorized\"}") } //NotFoundRoute return when route not found diff --git a/api/testutils/server.go b/api/testutils/server.go index a299005e..425951c9 100644 --- a/api/testutils/server.go +++ b/api/testutils/server.go @@ -15,6 +15,7 @@ import ( func GetAuthenticationConfig() config.AuthenticationConfig { return config.AuthenticationConfig{ + Enabled: true, Accounts: []config.AccountConfig{ { Name: "User", diff --git a/configuration/api.yaml b/configuration/api.yaml index e9799596..2325b4f3 100644 --- a/configuration/api.yaml +++ b/configuration/api.yaml @@ -8,6 +8,7 @@ storage: endpoints: - http://127.0.0.1:9200 authentication: + enabled: true accounts: - name: User password: Finala \ No newline at end of file From c01bbd0743cd7b1a511863f9fe8470f5c66f4f63 Mon Sep 17 00:00:00 2001 From: daniebrill <50454544+daniebrill@users.noreply.github.com> Date: Fri, 23 Jul 2021 15:50:12 +0200 Subject: [PATCH 58/68] Add authentication request + initial call in DataFactory.js --- ui/src/components/DataFactory.js | 7 ++++++ ui/src/services/authentification.service.js | 28 +++++++++++++++++++++ ui/src/services/request.service.js | 16 ++++++++++++ 3 files changed, 51 insertions(+) create mode 100644 ui/src/services/authentification.service.js diff --git a/ui/src/components/DataFactory.js b/ui/src/components/DataFactory.js index 1e90ccf0..5db0b10f 100644 --- a/ui/src/components/DataFactory.js +++ b/ui/src/components/DataFactory.js @@ -1,6 +1,7 @@ import React, { Fragment, useEffect } from "react"; import { connect } from "react-redux"; import PropTypes from "prop-types"; +import { AuthService } from "../services/authentification.service"; import { ResourcesService } from "services/resources.service"; import { SettingsService } from "services/settings.service"; import { titleDirective } from "utils/Title"; @@ -51,6 +52,12 @@ const DataFacotry = ({ setIsAppLoading, setIsScanning, }) => { + const checkAuthentication = async (username, password) => { + const response = await AuthService.Auth(username, password).catch(() => {}); + }; + useEffect(() => { + checkAuthentication("", ""); + }, []); /** * start fetching data from server * will load executions list diff --git a/ui/src/services/authentification.service.js b/ui/src/services/authentification.service.js new file mode 100644 index 00000000..8d8eac21 --- /dev/null +++ b/ui/src/services/authentification.service.js @@ -0,0 +1,28 @@ +import { http } from "./request.service"; +export const AuthService = { + Auth, +}; + +/** + * + * @param username {string} The users username + * @param password {string} The users password + */ +function Auth(username, password) { + const body = { + Username: username, + Password: password, + }; + const customRequestOptions = { + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }; + return http + .send("api/v1/login", `get`, customRequestOptions) + .then(this.handleAuthResponse) + .then((response) => { + return response; + }); +} diff --git a/ui/src/services/request.service.js b/ui/src/services/request.service.js index d8f683f7..07b6c373 100644 --- a/ui/src/services/request.service.js +++ b/ui/src/services/request.service.js @@ -16,6 +16,7 @@ class Http { request(url, action, customRequestOptions = {}) { let defaultRequestOptions = { method: action, + credentials: "include", }; merge(defaultRequestOptions, customRequestOptions); let fullUrl = ""; @@ -44,6 +45,21 @@ function handleResponse(response) { }); } +/** + * Manage http login request response + * + * @param {response} url url request + * @returns {object} + */ +function handleAuthResponse(response) { + return response.json().then((result) => { + if (response.status == 200 || response.status == 401) { + return result; + } + return Promise.reject(response); + }); +} + const HTTPRequests = new Http(); export const http = { send: HTTPRequests.request, From ab6405e640d11bf5702ff7b4c4d13e6165864f54 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Fri, 23 Jul 2021 17:36:42 +0200 Subject: [PATCH 59/68] # SOS-57 remove account charts with not data found --- ui/src/components/Dashboard/ResourcesChart.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/src/components/Dashboard/ResourcesChart.js b/ui/src/components/Dashboard/ResourcesChart.js index fa1fa8ce..8a56a6a1 100644 --- a/ui/src/components/Dashboard/ResourcesChart.js +++ b/ui/src/components/Dashboard/ResourcesChart.js @@ -179,6 +179,10 @@ const ResourcesChart = ({ return resource; }); + if (account && !sortedResources.length && !isResourceListLoading) { + return ; + } + return ( From 13c945af5044550236fb83d632220a974e071f55 Mon Sep 17 00:00:00 2001 From: DerOtt Date: Mon, 26 Jul 2021 13:10:00 +0200 Subject: [PATCH 60/68] Experimental first version of login.js --- ui/src/components/Dashboard/Index.js | 1 + ui/src/components/Dashboard/Login.js | 93 ++++++++++++++++++++++++++++ ui/src/components/DataFactory.js | 10 +++ ui/src/routes/index.js | 22 ++++--- 4 files changed, 117 insertions(+), 9 deletions(-) create mode 100644 ui/src/components/Dashboard/Login.js diff --git a/ui/src/components/Dashboard/Index.js b/ui/src/components/Dashboard/Index.js index 5d48e9b8..8b0928d6 100644 --- a/ui/src/components/Dashboard/Index.js +++ b/ui/src/components/Dashboard/Index.js @@ -90,6 +90,7 @@ DashboardIndex.propTypes = { const mapStateToProps = (state) => ({ currentResource: state.resources.currentResource, filters: state.filters.filters, + }); const mapDispatchToProps = (dispatch) => ({ diff --git a/ui/src/components/Dashboard/Login.js b/ui/src/components/Dashboard/Login.js new file mode 100644 index 00000000..316d5999 --- /dev/null +++ b/ui/src/components/Dashboard/Login.js @@ -0,0 +1,93 @@ +import React from 'react'; +import Button from '@material-ui/core/Button'; +import Container from '@material-ui/core/Container'; +import Grid from '@material-ui/core/Grid'; +import TextField from '@material-ui/core/TextField'; +import { makeStyles } from '@material-ui/core/styles'; +import {PropTypes} from "@material-ui/core"; +import { AuthService } from "../services/authentification.service"; +import { useForm } from 'react-hook-form'; + +const useStyles = makeStyles((theme) => ({ + container: { + padding: theme.spacing(3), + }, +}); + +interface FormData { + username: string; + password: string; +} + + +const Login = ({ + authRequired, + }) => { + const attemptLogin = (username, password) => { + if (AuthService.Auth( username,password) == 200){ + setAuthRequired(false) + } else { + //huh + } + }; + + const { handleSubmit, register } = useForm(); + + const classes = useStyles(); + + const onSubmit = handleSubmit((data) => { + attemptLogin(data.username, data.password) + }); + + return ( + +
+ + + + + + + + + + + + + + + +
+ + ) +} + +Login.defaultProps = {}; +Login.propTypes = { + authRequired: PropTypes.bool, +} + +const mapStateToProps = (state) => ({ + authRequired: state.accounts.authRequired, +}) + +const mapDispatchToProps = (dispatch) => ({ + setAuthRequired: (authRequired) => dispatch({type: "AUTH_REQUIRED", authRequired}), +}) diff --git a/ui/src/components/DataFactory.js b/ui/src/components/DataFactory.js index 5db0b10f..9616e01a 100644 --- a/ui/src/components/DataFactory.js +++ b/ui/src/components/DataFactory.js @@ -51,9 +51,13 @@ const DataFacotry = ({ resources, setIsAppLoading, setIsScanning, + setAuthRequired, }) => { const checkAuthentication = async (username, password) => { const response = await AuthService.Auth(username, password).catch(() => {}); + if (response.ok) { + setAuthRequired(false); + } }; useEffect(() => { checkAuthentication("", ""); @@ -291,6 +295,7 @@ DataFacotry.propTypes = { setResources: PropTypes.func, setCurrentResourceData: PropTypes.func, setCurrentExecution: PropTypes.func, + setAuthRequired: PropTypes.func, currentResource: PropTypes.string, accounts: PropTypes.object, @@ -301,6 +306,8 @@ DataFacotry.propTypes = { setScanning: PropTypes.func, isScanning: PropTypes.bool, + + authRequired: PropTypes.bool, }; const mapStateToProps = (state) => ({ @@ -310,6 +317,7 @@ const mapStateToProps = (state) => ({ currentExecution: state.executions.current, filters: state.filters.filters, isScanning: state.executions.isScanning, + authRequired: state.accounts.authRequired, }); const mapDispatchToProps = (dispatch) => ({ @@ -326,6 +334,8 @@ const mapDispatchToProps = (dispatch) => ({ setCurrentExecution: (id) => dispatch({ type: "EXECUTION_SELECTED", id }), setCurrentResourceData: (data) => dispatch({ type: "SET_CURRENT_RESOURCE_DATA", data }), + setAuthRequired: (authRequired) => + dispatch({ type: "AUTH_REQUIRED", authRequired }), }); export default connect(mapStateToProps, mapDispatchToProps)(DataFacotry); diff --git a/ui/src/routes/index.js b/ui/src/routes/index.js index 038295a8..e1bb359e 100644 --- a/ui/src/routes/index.js +++ b/ui/src/routes/index.js @@ -1,14 +1,15 @@ import React from "react"; import { connect } from "react-redux"; -import { Route, Switch } from "react-router"; import PropTypes from "prop-types"; -import Dashboard from "../components/Dashboard/Index"; import PageLoader from "../components/PageLoader"; -import NotFound from "../components/NotFound"; -import NoData from "../components/NoData"; import DataFactory from "../components/DataFactory"; +import Login from "../components/Dashboard/Login"; -import { CssBaseline, makeStyles, Box } from "@material-ui/core"; +import { Box, CssBaseline, makeStyles } from "@material-ui/core"; +import NoData from "../components/NoData"; +import {Dashboard} from "@material-ui/icons"; +import NotFound from "../components/NotFound"; +import {Route, Switch} from "react-router"; const useStyles = makeStyles(() => ({ root: { @@ -29,7 +30,7 @@ const useStyles = makeStyles(() => ({ * @param {bool} isAppLoading App loading state * @param {array} executions Executions list */ -const RouterIndex = ({ isAppLoading, executions }) => { +const RouterIndex = ({ isAppLoading, executions, authRequired }) => { const classes = useStyles(); return ( @@ -38,9 +39,10 @@ const RouterIndex = ({ isAppLoading, executions }) => {
- {isAppLoading && } - {!isAppLoading && !executions.length && } - {!isAppLoading && executions.length > 0 && ( + {authRequired && } + {!authRequired && isAppLoading && } + {!authRequired && !isAppLoading && !executions.length && } + {!authRequired && !isAppLoading && executions.length > 0 && ( @@ -57,6 +59,7 @@ const RouterIndex = ({ isAppLoading, executions }) => { const mapStateToProps = (state) => ({ executions: state.executions.list, isAppLoading: state.executions.isAppLoading, + authRequired: state.accounts.authRequired, }); const mapDispatchToProps = () => ({}); @@ -66,6 +69,7 @@ RouterIndex.propTypes = { isAppLoading: PropTypes.bool, executions: PropTypes.array, setCurrentExecution: PropTypes.func, + authRequired: PropTypes.bool, }; export default connect(mapStateToProps, mapDispatchToProps)(RouterIndex); From 3864503e842c8a9ee50a9e15de4c121ab873612d Mon Sep 17 00:00:00 2001 From: daniebrill <50454544+daniebrill@users.noreply.github.com> Date: Tue, 27 Jul 2021 14:56:16 +0200 Subject: [PATCH 61/68] Adjust api for cookie requests --- api/config/config.go | 5 +++++ api/config/testutil/mock/config.yaml | 3 ++- api/route.go | 4 ++++ api/server.go | 12 ++++++++---- api/server_test.go | 5 +++-- api/testutils/server.go | 4 ++++ cmd/api.go | 2 +- configuration/api.yaml | 3 ++- 8 files changed, 29 insertions(+), 9 deletions(-) diff --git a/api/config/config.go b/api/config/config.go index 5200bfe7..7059cae8 100644 --- a/api/config/config.go +++ b/api/config/config.go @@ -32,9 +32,14 @@ type AuthenticationConfig struct { Accounts []AccountConfig `yaml:"accounts"` } +type UIServerConfig struct { + Address string `yaml:"address"` +} + // APIConfig present the application config type APIConfig struct { LogLevel string `yaml:"log_level"` + UIServer UIServerConfig `yaml:"ui_server"` Storage StorageConfig `yaml:"storage"` Authentication AuthenticationConfig `yaml:"authentication"` } diff --git a/api/config/testutil/mock/config.yaml b/api/config/testutil/mock/config.yaml index af95054a..8dabde3c 100644 --- a/api/config/testutil/mock/config.yaml +++ b/api/config/testutil/mock/config.yaml @@ -1,6 +1,7 @@ --- log_level: info - +ui_server: + address: http://127.0.0.1:8080 storage: elasticsearch: index: general diff --git a/api/route.go b/api/route.go index 7d4096ba..b14beb0b 100644 --- a/api/route.go +++ b/api/route.go @@ -184,6 +184,10 @@ func (server *Server) DetectEvents(resp http.ResponseWriter, req *http.Request) } func (server *Server) Login(resp http.ResponseWriter, req *http.Request) { + if req.Method == http.MethodOptions { + server.JSONWrite(resp, http.StatusOK, nil) + return + } if server.authentication.Enabled { buf, bodyErr := ioutil.ReadAll(req.Body) diff --git a/api/server.go b/api/server.go index 0557ae3a..67c4b21e 100644 --- a/api/server.go +++ b/api/server.go @@ -33,17 +33,21 @@ type Server struct { } // NewServer returns a new Server -func NewServer(port int, storage storage.StorageDescriber, version version.VersionManagerDescriptor, auth config.AuthenticationConfig) *Server { +func NewServer(port int, storage storage.StorageDescriber, version version.VersionManagerDescriptor, auth config.AuthenticationConfig, allowedOrigin string) *Server { router := mux.NewRouter() - corsObj := handlers.AllowedOrigins([]string{"*"}) + corsObjects := []handlers.CORSOption{} + corsObjects = append(corsObjects, handlers.AllowedOrigins([]string{allowedOrigin})) + corsObjects = append(corsObjects, handlers.AllowedMethods([]string{"GET", "POST", "OPTIONS"})) + corsObjects = append(corsObjects, handlers.AllowedHeaders([]string{"Content-Type"})) + corsObjects = append(corsObjects, handlers.AllowCredentials()) return &Server{ router: router, storage: storage, version: version, authentication: auth, httpserver: &http.Server{ - Handler: handlers.CORS(corsObj)(router), + Handler: handlers.CORS(corsObjects...)(router), Addr: fmt.Sprintf("0.0.0.0:%d", port), }, } @@ -89,7 +93,7 @@ func (server *Server) BindEndpoints() { server.router.HandleFunc("/api/v1/trends/{type}", server.middleware(http.HandlerFunc(server.GetResourceTrends))).Methods("GET") server.router.HandleFunc("/api/v1/tags/{executionID}", server.middleware(http.HandlerFunc(server.GetExecutionTags))).Methods("GET") server.router.HandleFunc("/api/v1/detect-events/{executionID}", server.DetectEvents).Methods("POST") - server.router.HandleFunc("/api/v1/login", server.Login).Methods("GET") + server.router.HandleFunc("/api/v1/login", server.Login).Methods("POST", "OPTIONS") server.router.HandleFunc("/api/v1/version", server.VersionHandler).Methods("GET") server.router.HandleFunc("/api/v1/health", server.HealthCheckHandler).Methods("GET") server.router.HandleFunc("/api/v1/report/{executionID}", server.GetReport).Methods("GET") diff --git a/api/server_test.go b/api/server_test.go index c0f9b8b1..428c6a6d 100644 --- a/api/server_test.go +++ b/api/server_test.go @@ -21,8 +21,9 @@ func MockServer() (*api.Server, *testutils.MockStorage) { mockStorage := testutils.NewMockStorage() mockAuthenticationConfig := testutils.GetAuthenticationConfig() + allowedOrigin := testutils.GetAllowedOrigin() - server := api.NewServer(9090, mockStorage, version, mockAuthenticationConfig) + server := api.NewServer(9090, mockStorage, version, mockAuthenticationConfig, allowedOrigin) return server, mockStorage } @@ -545,7 +546,7 @@ func TestLogin(t *testing.T) { t.Fatal(err) } - req, err := http.NewRequest("GET", test.endpoint, bytes.NewBuffer(buf)) + req, err := http.NewRequest("POST", test.endpoint, bytes.NewBuffer(buf)) if err != nil { t.Fatal(err) } diff --git a/api/testutils/server.go b/api/testutils/server.go index 425951c9..03c56f9a 100644 --- a/api/testutils/server.go +++ b/api/testutils/server.go @@ -25,6 +25,10 @@ func GetAuthenticationConfig() config.AuthenticationConfig { } } +func GetAllowedOrigin() string { + return "http://127.0.0.1:8080" +} + func GetTestCookie() *http.Cookie { expTime := time.Now().Add(time.Minute * 5) diff --git a/cmd/api.go b/cmd/api.go index b8efc688..23c0e212 100644 --- a/cmd/api.go +++ b/cmd/api.go @@ -42,7 +42,7 @@ var apiServer = &cobra.Command{ os.Exit(1) } - apiManager := api.NewServer(port, storage, versionManager, configStruct.Authentication) + apiManager := api.NewServer(port, storage, versionManager, configStruct.Authentication, configStruct.UIServer.Address) apiStopper := serverutil.RunAll(apiManager).StopFunc diff --git a/configuration/api.yaml b/configuration/api.yaml index 2325b4f3..a5ce2c7e 100644 --- a/configuration/api.yaml +++ b/configuration/api.yaml @@ -1,6 +1,7 @@ --- log_level: info - +ui_server: + address: http://127.0.0.1:8080 storage: elasticsearch: username: "" From 0321b8db7439e3bd66b9a588e43af10dfa8abd89 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Tue, 27 Jul 2021 17:25:24 +0200 Subject: [PATCH 62/68] change token library delete unnecessary console define SameSite --- api/route.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/api/route.go b/api/route.go index b14beb0b..9fdb3996 100644 --- a/api/route.go +++ b/api/route.go @@ -5,7 +5,7 @@ import ( "finala/api/httpparameters" "finala/api/storage" "fmt" - "github.com/dgrijalva/jwt-go" + "github.com/golang-jwt/jwt" "io/ioutil" "net/http" "net/url" @@ -220,9 +220,10 @@ func (server *Server) Login(resp http.ResponseWriter, req *http.Request) { } cookie := http.Cookie{ - Name: "jwt", - Value: token, - Expires: expTime, + Name: "jwt", + Value: token, + Expires: expTime, + SameSite: http.SameSiteNoneMode, } http.SetCookie(resp, &cookie) @@ -245,9 +246,10 @@ func (server *Server) Login(resp http.ResponseWriter, req *http.Request) { } cookie := http.Cookie{ - Name: "jwt", - Value: token, - Expires: expTime, + Name: "jwt", + Value: token, + Expires: expTime, + SameSite: http.SameSiteLaxMode, } http.SetCookie(resp, &cookie) @@ -277,7 +279,7 @@ func (server *Server) VersionHandler(resp http.ResponseWriter, req *http.Request func (server *Server) middleware(next http.Handler) http.HandlerFunc { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { cookie, err := req.Cookie("jwt") - log.Println(cookie) + if err != nil { server.JSONWrite(resp, http.StatusUnauthorized, HttpErrorResponse{Error: "Authorize cookie not found"}) return From aace35201d17aa6301284a098a2f22c55f18a8b8 Mon Sep 17 00:00:00 2001 From: daniebrill <50454544+daniebrill@users.noreply.github.com> Date: Tue, 27 Jul 2021 21:56:40 +0200 Subject: [PATCH 63/68] Add UI Authentication functionality --- go.mod | 1 + go.sum | 2 + ui/src/components/Dashboard/Index.js | 1 - ui/src/components/Dashboard/Login.js | 106 +++++++++--------- ui/src/components/DataFactory.js | 33 +++--- ui/src/reducers/auth.reducer.js | 18 +++ ui/src/reducers/index.js | 2 + ui/src/routes/index.js | 6 +- ...n.service.js => authentication.service.js} | 4 +- ui/src/services/request.service.js | 46 ++++++-- 10 files changed, 137 insertions(+), 82 deletions(-) create mode 100644 ui/src/reducers/auth.reducer.js rename ui/src/services/{authentification.service.js => authentication.service.js} (84%) diff --git a/go.mod b/go.mod index 36a815bf..f6c3aea6 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/aws/aws-sdk-go v1.31.3 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dustin/go-humanize v1.0.0 + github.com/golang-jwt/jwt v3.2.1+incompatible github.com/gorilla/handlers v1.4.2 github.com/gorilla/mux v1.7.4 github.com/kr/pretty v0.2.0 // indirect diff --git a/go.sum b/go.sum index 8fc491d2..65ff8f20 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= diff --git a/ui/src/components/Dashboard/Index.js b/ui/src/components/Dashboard/Index.js index 8b0928d6..5d48e9b8 100644 --- a/ui/src/components/Dashboard/Index.js +++ b/ui/src/components/Dashboard/Index.js @@ -90,7 +90,6 @@ DashboardIndex.propTypes = { const mapStateToProps = (state) => ({ currentResource: state.resources.currentResource, filters: state.filters.filters, - }); const mapDispatchToProps = (dispatch) => ({ diff --git a/ui/src/components/Dashboard/Login.js b/ui/src/components/Dashboard/Login.js index 316d5999..334b3cab 100644 --- a/ui/src/components/Dashboard/Login.js +++ b/ui/src/components/Dashboard/Login.js @@ -1,93 +1,99 @@ -import React from 'react'; -import Button from '@material-ui/core/Button'; -import Container from '@material-ui/core/Container'; -import Grid from '@material-ui/core/Grid'; -import TextField from '@material-ui/core/TextField'; -import { makeStyles } from '@material-ui/core/styles'; -import {PropTypes} from "@material-ui/core"; -import { AuthService } from "../services/authentification.service"; -import { useForm } from 'react-hook-form'; +import React from "react"; +import Button from "@material-ui/core/Button"; +import Grid from "@material-ui/core/Grid"; +import TextField from "@material-ui/core/TextField"; +import { makeStyles } from "@material-ui/core/styles"; +import PropTypes from "prop-types"; +import { AuthService } from "../../services/authentication.service"; +import { connect } from "react-redux"; +import Logo from "../Logo"; +import Box from "@material-ui/core/Box"; +import { Fragment, useState } from "react"; const useStyles = makeStyles((theme) => ({ container: { padding: theme.spacing(3), }, -}); - -interface FormData { - username: string; - password: string; -} + box: { + textAlign: "center", + marginRight: "auto", + marginLeft: "auto", + maxWidth: "600px", + }, +})); +/** + * @param {boolean} setAuthRequired tells if authentication is needed + */ +const Login = ({ setAuthRequired }) => { + const classes = useStyles(); + const [username, setUsername] = useState(""); + const [password, setPassword] = useState(""); -const Login = ({ - authRequired, - }) => { - const attemptLogin = (username, password) => { - if (AuthService.Auth( username,password) == 200){ - setAuthRequired(false) - } else { - //huh + const onSubmit = async () => { + const ok = await AuthService.Auth(username, password).catch(() => false); + if (ok) { + setAuthRequired(false); } }; - const { handleSubmit, register } = useForm(); - - const classes = useStyles(); - - const onSubmit = handleSubmit((data) => { - attemptLogin(data.username, data.password) - }); - return ( - -
+ + + setUsername(e.target.value)} fullWidth - inputRef={register} - label="username" - name="eusername" + label="Username" + name="username" size="small" variant="outlined" + value={username} /> setPassword(e.target.value)} fullWidth - inputRef={register} label="Password" name="password" size="small" type="password" variant="outlined" + value={password} /> - - -
- ) -} + + + ); +}; Login.defaultProps = {}; Login.propTypes = { - authRequired: PropTypes.bool, -} - -const mapStateToProps = (state) => ({ - authRequired: state.accounts.authRequired, -}) + setAuthRequired: PropTypes.func, +}; const mapDispatchToProps = (dispatch) => ({ - setAuthRequired: (authRequired) => dispatch({type: "AUTH_REQUIRED", authRequired}), -}) + setAuthRequired: (authRequired) => + dispatch({ type: "SET_AUTH_REQUIRED", authRequired }), +}); + +export default connect(null, mapDispatchToProps)(Login); diff --git a/ui/src/components/DataFactory.js b/ui/src/components/DataFactory.js index 9616e01a..915f0d5d 100644 --- a/ui/src/components/DataFactory.js +++ b/ui/src/components/DataFactory.js @@ -1,7 +1,7 @@ import React, { Fragment, useEffect } from "react"; import { connect } from "react-redux"; import PropTypes from "prop-types"; -import { AuthService } from "../services/authentification.service"; +import { AuthService } from "../services/authentication.service"; import { ResourcesService } from "services/resources.service"; import { SettingsService } from "services/settings.service"; import { titleDirective } from "utils/Title"; @@ -33,6 +33,7 @@ let lastFiltersSearched = "[]"; * * @param {func} setIsAppLoading Update App IsLoading status * @param {func} setIsScanning Update scanning status + * @param {func} setAuthRequired Update, if authentication is required * } */ const DataFacotry = ({ @@ -53,24 +54,27 @@ const DataFacotry = ({ setIsScanning, setAuthRequired, }) => { - const checkAuthentication = async (username, password) => { - const response = await AuthService.Auth(username, password).catch(() => {}); - if (response.ok) { - setAuthRequired(false); - } - }; - useEffect(() => { - checkAuthentication("", ""); - }, []); /** * start fetching data from server + * will check for enabled authentication * will load executions list */ const init = async () => { await SettingsService.GetSettings().catch(() => false); + checkAuthentication(); fetchData(); }; + /** + * checks, if authentication is required/enabled + */ + const checkAuthentication = async () => { + const authenticated = await AuthService.Auth("", "").catch(() => false); + if (authenticated) { + setAuthRequired(false); + } + }; + /** * Fetch data from server: executionsList & update currentExecution */ @@ -79,7 +83,6 @@ const DataFacotry = ({ const executionsList = await ResourcesService.GetExecutions().catch( () => [] ); - setExecutions(executionsList); setIsAppLoading(false); if (!executionsList.length) { @@ -120,7 +123,7 @@ const DataFacotry = ({ }; /** - * Triggered every-time executionId/filters changes and re-load resources list + * Triggered every-time executionId/filters changes and re-load resources list and accounts list * @param {string} currentExecution Current Selected Execution * @param {array} filters Filters List */ @@ -306,8 +309,6 @@ DataFacotry.propTypes = { setScanning: PropTypes.func, isScanning: PropTypes.bool, - - authRequired: PropTypes.bool, }; const mapStateToProps = (state) => ({ @@ -317,7 +318,6 @@ const mapStateToProps = (state) => ({ currentExecution: state.executions.current, filters: state.filters.filters, isScanning: state.executions.isScanning, - authRequired: state.accounts.authRequired, }); const mapDispatchToProps = (dispatch) => ({ @@ -334,8 +334,7 @@ const mapDispatchToProps = (dispatch) => ({ setCurrentExecution: (id) => dispatch({ type: "EXECUTION_SELECTED", id }), setCurrentResourceData: (data) => dispatch({ type: "SET_CURRENT_RESOURCE_DATA", data }), - setAuthRequired: (authRequired) => - dispatch({ type: "AUTH_REQUIRED", authRequired }), + setAuthRequired: (data) => dispatch({ type: "SET_AUTH_REQUIRED", data }), }); export default connect(mapStateToProps, mapDispatchToProps)(DataFacotry); diff --git a/ui/src/reducers/auth.reducer.js b/ui/src/reducers/auth.reducer.js new file mode 100644 index 00000000..9fcce5ea --- /dev/null +++ b/ui/src/reducers/auth.reducer.js @@ -0,0 +1,18 @@ +const initialState = { + authRequired: true, +}; + +/** + * @param {object} state module state + * @param {object} action to apply on state + * @returns {object} new copy of state + */ +export function auth(state = initialState, action) { + switch (action.type) { + case "SET_AUTH_REQUIRED": + state.authRequired = action.data; + return { ...state }; + default: + return state; + } +} diff --git a/ui/src/reducers/index.js b/ui/src/reducers/index.js index ad94d29c..b7395814 100755 --- a/ui/src/reducers/index.js +++ b/ui/src/reducers/index.js @@ -1,5 +1,6 @@ import { combineReducers } from "redux"; import { connectRouter } from "connected-react-router"; +import { auth } from "../reducers/auth.reducer"; import { accounts } from "../reducers/accounts.reducer"; import { resources } from "../reducers/resources.reducer"; import { executions } from "../reducers/executions.reducer"; @@ -7,6 +8,7 @@ import { filters } from "../reducers/filters.reducer"; const rootReducer = (history) => combineReducers({ + auth, accounts, resources, executions, diff --git a/ui/src/routes/index.js b/ui/src/routes/index.js index e1bb359e..f1a7ee64 100644 --- a/ui/src/routes/index.js +++ b/ui/src/routes/index.js @@ -7,9 +7,9 @@ import Login from "../components/Dashboard/Login"; import { Box, CssBaseline, makeStyles } from "@material-ui/core"; import NoData from "../components/NoData"; -import {Dashboard} from "@material-ui/icons"; +import Dashboard from "../components/Dashboard/Index"; import NotFound from "../components/NotFound"; -import {Route, Switch} from "react-router"; +import { Route, Switch } from "react-router"; const useStyles = makeStyles(() => ({ root: { @@ -59,7 +59,7 @@ const RouterIndex = ({ isAppLoading, executions, authRequired }) => { const mapStateToProps = (state) => ({ executions: state.executions.list, isAppLoading: state.executions.isAppLoading, - authRequired: state.accounts.authRequired, + authRequired: state.auth.authRequired, }); const mapDispatchToProps = () => ({}); diff --git a/ui/src/services/authentification.service.js b/ui/src/services/authentication.service.js similarity index 84% rename from ui/src/services/authentification.service.js rename to ui/src/services/authentication.service.js index 8d8eac21..184dcd4d 100644 --- a/ui/src/services/authentification.service.js +++ b/ui/src/services/authentication.service.js @@ -4,7 +4,7 @@ export const AuthService = { }; /** - * + * send authentication request to api * @param username {string} The users username * @param password {string} The users password */ @@ -20,7 +20,7 @@ function Auth(username, password) { body: JSON.stringify(body), }; return http - .send("api/v1/login", `get`, customRequestOptions) + .sendAuth("api/v1/login", `post`, customRequestOptions) .then(this.handleAuthResponse) .then((response) => { return response; diff --git a/ui/src/services/request.service.js b/ui/src/services/request.service.js index 07b6c373..a71ac158 100644 --- a/ui/src/services/request.service.js +++ b/ui/src/services/request.service.js @@ -10,7 +10,7 @@ class Http { * * @param {string} url url request * @param {action} action method request (GET,POST,etc.) - * @param {object} url customRequestOptions custom request options + * @param {object} customRequestOptions customRequestOptions custom request options * @returns {Promise} */ request(url, action, customRequestOptions = {}) { @@ -28,13 +28,36 @@ class Http { return fetch(`${fullUrl}`, defaultRequestOptions).then(handleResponse); } + + /** + * Making authentication request + * + * @param {string} url url request + * @param {action} action method request (GET,POST,etc.) + * @param {object} customRequestOptions customRequestOptions custom request options + * @returns {Promise} + */ + requestAuth(url, action, customRequestOptions) { + let defaultRequestOptions = { + method: action, + credentials: "include", + }; + merge(defaultRequestOptions, customRequestOptions); + let fullUrl = ""; + if (url.startsWith("http")) { + fullUrl = url; + } else { + fullUrl = `${this.baseURL}/${url}`; + } + return fetch(`${fullUrl}`, defaultRequestOptions).then(handleAuthResponse); + } } /** * Manage http request response * * @param {response} url url request - * @returns {object} + * @returns {Promise} */ function handleResponse(response) { return response.json().then((result) => { @@ -46,15 +69,19 @@ function handleResponse(response) { } /** - * Manage http login request response + * Manage http login response * - * @param {response} url url request - * @returns {object} - */ + * @param {response} response request response + * @returns {boolean, Promise} authentication correct or Promise, if rejected + * */ function handleAuthResponse(response) { - return response.json().then((result) => { - if (response.status == 200 || response.status == 401) { - return result; + return response.text().then(() => { + if (response.status === 200) { + return true; + } else { + if (response.status === 401) { + return false; + } } return Promise.reject(response); }); @@ -63,4 +90,5 @@ function handleAuthResponse(response) { const HTTPRequests = new Http(); export const http = { send: HTTPRequests.request, + sendAuth: HTTPRequests.requestAuth, }; From 81fe9fc9dc45d6dfcaee62a9748fef6fa69e1a0e Mon Sep 17 00:00:00 2001 From: daniebrill Date: Tue, 27 Jul 2021 22:02:37 +0200 Subject: [PATCH 64/68] Replace unmaintained auth library in api server test files --- api/testutils/server.go | 2 +- go.mod | 1 - go.sum | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/api/testutils/server.go b/api/testutils/server.go index 03c56f9a..335c6cc0 100644 --- a/api/testutils/server.go +++ b/api/testutils/server.go @@ -2,7 +2,7 @@ package testutils import ( "finala/api/config" - "github.com/dgrijalva/jwt-go" + "github.com/golang-jwt/jwt" "net" "net/http" "strings" diff --git a/go.mod b/go.mod index f6c3aea6..3ac71c71 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.13 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/aws/aws-sdk-go v1.31.3 - github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dustin/go-humanize v1.0.0 github.com/golang-jwt/jwt v3.2.1+incompatible github.com/gorilla/handlers v1.4.2 diff --git a/go.sum b/go.sum index 65ff8f20..77c7e719 100644 --- a/go.sum +++ b/go.sum @@ -22,7 +22,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= From f8adc70e6e5ec579e78e6b84cdbc0437d2fe4f69 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 28 Jul 2021 15:32:55 +0200 Subject: [PATCH 65/68] set account to filters if an account specific Chart is clicked --- ui/src/components/Dashboard/ResourcesChart.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ui/src/components/Dashboard/ResourcesChart.js b/ui/src/components/Dashboard/ResourcesChart.js index 8a56a6a1..92a76515 100644 --- a/ui/src/components/Dashboard/ResourcesChart.js +++ b/ui/src/components/Dashboard/ResourcesChart.js @@ -47,6 +47,7 @@ const useStyles = makeStyles(() => ({ const ResourcesChart = ({ resources, filters, + setFilters, isResourceListLoading, addFilter, setResource, @@ -81,6 +82,18 @@ const ResourcesChart = ({ const res = sortedResources; const selectedResource = res[dataPointIndex]; setSelectedResource(selectedResource); + if (account) { + const nfilters = filters.filter( + (filter) => filter.type !== "account" + ); + setFilters(nfilters); + const filter = { + title: `Account:${account}`, + id: `account:${account}`, + type: "account", + }; + addFilter(filter); + } }, }, }, @@ -224,6 +237,7 @@ ResourcesChart.defaultProps = {}; ResourcesChart.propTypes = { resources: PropTypes.object, filters: PropTypes.array, + setFilters: PropTypes.func, isResourceListLoading: PropTypes.bool, addFilter: PropTypes.func, setResource: PropTypes.func, @@ -239,6 +253,7 @@ const mapStateToProps = (state) => ({ }); const mapDispatchToProps = (dispatch) => ({ + setFilters: (data) => dispatch({ type: "SET_FILTERS", data }), addFilter: (data) => dispatch({ type: "ADD_FILTER", data }), setResource: (data) => dispatch({ type: "SET_RESOURCE", data }), }); From 16da0d8b1144ed611eae818ba21e818e0aadfb9d Mon Sep 17 00:00:00 2001 From: daniebrill Date: Wed, 28 Jul 2021 15:40:17 +0200 Subject: [PATCH 66/68] Show error message for wrong credentials + login with enter --- ui/src/components/Dashboard/Login.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ui/src/components/Dashboard/Login.js b/ui/src/components/Dashboard/Login.js index 334b3cab..77903675 100644 --- a/ui/src/components/Dashboard/Login.js +++ b/ui/src/components/Dashboard/Login.js @@ -20,6 +20,9 @@ const useStyles = makeStyles((theme) => ({ marginLeft: "auto", maxWidth: "600px", }, + error: { + color: "red", + }, })); /** @@ -29,11 +32,15 @@ const Login = ({ setAuthRequired }) => { const classes = useStyles(); const [username, setUsername] = useState(""); const [password, setPassword] = useState(""); + const [errorMessage, setErrorMessage] = useState(""); const onSubmit = async () => { + setErrorMessage(""); const ok = await AuthService.Auth(username, password).catch(() => false); if (ok) { setAuthRequired(false); + } else { + setErrorMessage("Login Failed"); } }; @@ -47,6 +54,11 @@ const Login = ({ setAuthRequired }) => { setUsername(e.target.value)} + onKeyPress={(e) => { + if (e.key == "Enter") { + onSubmit(); + } + }} fullWidth label="Username" name="username" @@ -58,6 +70,11 @@ const Login = ({ setAuthRequired }) => { setPassword(e.target.value)} + onKeyPress={(e) => { + if (e.key == "Enter") { + onSubmit(); + } + }} fullWidth label="Password" name="password" @@ -80,6 +97,9 @@ const Login = ({ setAuthRequired }) => { Log in + + {errorMessage} + From a3fdc4fa3afc7c5b265a38f872c73597ab0820b2 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 28 Jul 2021 16:40:34 +0200 Subject: [PATCH 67/68] authentication needed for report samesite from a cookie is laxmode --- api/route.go | 2 +- api/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/route.go b/api/route.go index 9fdb3996..9a3bf220 100644 --- a/api/route.go +++ b/api/route.go @@ -223,7 +223,7 @@ func (server *Server) Login(resp http.ResponseWriter, req *http.Request) { Name: "jwt", Value: token, Expires: expTime, - SameSite: http.SameSiteNoneMode, + SameSite: http.SameSiteLaxMode, } http.SetCookie(resp, &cookie) diff --git a/api/server.go b/api/server.go index 67c4b21e..ec81626e 100644 --- a/api/server.go +++ b/api/server.go @@ -92,11 +92,11 @@ func (server *Server) BindEndpoints() { server.router.HandleFunc("/api/v1/resources/{type}", server.middleware(http.HandlerFunc(server.GetResourceData))).Methods("GET") server.router.HandleFunc("/api/v1/trends/{type}", server.middleware(http.HandlerFunc(server.GetResourceTrends))).Methods("GET") server.router.HandleFunc("/api/v1/tags/{executionID}", server.middleware(http.HandlerFunc(server.GetExecutionTags))).Methods("GET") + server.router.HandleFunc("/api/v1/report/{executionID}", server.middleware(http.HandlerFunc(server.GetReport))).Methods("GET") server.router.HandleFunc("/api/v1/detect-events/{executionID}", server.DetectEvents).Methods("POST") server.router.HandleFunc("/api/v1/login", server.Login).Methods("POST", "OPTIONS") server.router.HandleFunc("/api/v1/version", server.VersionHandler).Methods("GET") server.router.HandleFunc("/api/v1/health", server.HealthCheckHandler).Methods("GET") - server.router.HandleFunc("/api/v1/report/{executionID}", server.GetReport).Methods("GET") server.router.NotFoundHandler = http.HandlerFunc(server.NotFoundRoute) } From 21302aeef4705f555a1568df35d0f89c2047b345 Mon Sep 17 00:00:00 2001 From: Jackafive753 Date: Wed, 28 Jul 2021 17:38:21 +0200 Subject: [PATCH 68/68] position test method --- api/server_test.go | 96 +++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/api/server_test.go b/api/server_test.go index 428c6a6d..dd194cc5 100644 --- a/api/server_test.go +++ b/api/server_test.go @@ -293,6 +293,54 @@ func TestGetAccounts(t *testing.T) { } } +func TestLogin(t *testing.T) { + ms, _ := MockServer() + ms.BindEndpoints() + ms.Serve() + + type testAccount struct { + Username string + Password string + } + + testCases := []struct { + endpoint string + expectedStatusCode int + BodyRequest testAccount + }{ + {"/api/v1/login", http.StatusOK, testAccount{Username: "User", Password: "Finala"}}, + {"/api/v1/login", http.StatusUnauthorized, testAccount{Username: "", Password: ""}}, + } + + for _, test := range testCases { + t.Run(test.endpoint, func(t *testing.T) { + rr := httptest.NewRecorder() + buf, err := json.Marshal(test.BodyRequest) + if err != nil { + t.Fatal(err) + } + + req, err := http.NewRequest("POST", test.endpoint, bytes.NewBuffer(buf)) + if err != nil { + t.Fatal(err) + } + + ms.Router().ServeHTTP(rr, req) + if rr.Code != test.expectedStatusCode { + t.Fatalf("handler returned wrong status code: got %v want %v", rr.Code, test.expectedStatusCode) + } + + if test.expectedStatusCode == http.StatusOK { + if cookie := rr.Header().Get("Set-Cookie"); cookie == "" { + t.Fatalf("unexpected error, got %s expected jwt={SomeValue}", cookie) + } + } + + }) + } + +} + func TestSave(t *testing.T) { ms, mockStorage := MockServer() ms.BindEndpoints() @@ -518,51 +566,3 @@ func TestVersion(t *testing.T) { } } - -func TestLogin(t *testing.T) { - ms, _ := MockServer() - ms.BindEndpoints() - ms.Serve() - - type testAccount struct { - Username string - Password string - } - - testCases := []struct { - endpoint string - expectedStatusCode int - BodyRequest testAccount - }{ - {"/api/v1/login", http.StatusOK, testAccount{Username: "User", Password: "Finala"}}, - {"/api/v1/login", http.StatusUnauthorized, testAccount{Username: "", Password: ""}}, - } - - for _, test := range testCases { - t.Run(test.endpoint, func(t *testing.T) { - rr := httptest.NewRecorder() - buf, err := json.Marshal(test.BodyRequest) - if err != nil { - t.Fatal(err) - } - - req, err := http.NewRequest("POST", test.endpoint, bytes.NewBuffer(buf)) - if err != nil { - t.Fatal(err) - } - - ms.Router().ServeHTTP(rr, req) - if rr.Code != test.expectedStatusCode { - t.Fatalf("handler returned wrong status code: got %v want %v", rr.Code, test.expectedStatusCode) - } - - if test.expectedStatusCode == http.StatusOK { - if cookie := rr.Header().Get("Set-Cookie"); cookie == "" { - t.Fatalf("unexpected error, got %s expected jwt={SomeValue}", cookie) - } - } - - }) - } - -}