diff --git a/README.md b/README.md index 8cf2f32..b4961bc 100644 --- a/README.md +++ b/README.md @@ -64,14 +64,14 @@ global: port: ":8087" metric_path: "/metrics" scrape_batch_size: 300 - + resource_sync_interval_minutes: 20 # Update frequency of resource information: resource information is updated every 180 minutes by default; If this parameter is set to a value less than 10 minutes, the information is updated every 10 minutes. + ep_ids: "xxx1,xxx2" # This is optional. Filter resources by enterpries project, cloudeye-exporter will get all resources when this is empty, if you need multiple enterprise project, use comma split them. auth: auth_url: "https://iam.{region_id}.myhuaweicloud.com/v3" project_name: "{project_name}" access_key: "{access_key}" secret_key: "{secret_key}" region: "{region}" - ``` ## Example of endpoint file(endpoints.yml) diff --git a/README_cn.md b/README_cn.md index 7f63594..95ce149 100644 --- a/README_cn.md +++ b/README_cn.md @@ -83,6 +83,8 @@ tar -xzvf cloudeye-exporter.v2.0.5.tar.gz global: port: ":8087" # 监听端口 :8087代表在全部网络接口上开启监听8087端口，限定内部访问也可以指定IP例如：192.168.1.100:8087 scrape_batch_size: 300 + resource_sync_interval_minutes: 20 # 资源信息更新频率：默认180分钟更新一次；该配置值小于10分钟，将以10分钟1次为资源信息更新频率 + ep_ids: "xxx1,xxx2" # 可选配置，根据企业项目ID过滤资源，不配置默认查询所有资源的指标，多个ID使用英文逗号进行分割。 auth: auth_url: "https://iam.{region_id}.myhuaweicloud.com/v3" project_name: "cn-north-1" # 华为云项目名称，可以在“华为云->统一身份认证服务->项目”中查看 diff --git a/clouds.yml b/clouds.yml index b6726a6..c823002 100644 --- a/clouds.yml +++ b/clouds.yml @@ -1,7 +1,7 @@ global: prefix: "huaweicloud" scrape_batch_size: 300 - + resource_sync_interval_minutes: 180 auth: auth_url: "https://iam.xxx.yyy.com/v3" project_name: "{project_name}" diff --git a/collector/apic.go b/collector/apic.go index 1f9d569..f9404e1 100644 --- a/collector/apic.go +++ b/collector/apic.go @@ -4,12 +4,13 @@ import ( "strings" "time" - "github.com/huaweicloud/cloudeye-exporter/logs" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/config" apig "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/apig/v2" "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/apig/v2/model" cesmodel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) var apicInfo serversInfo @@ -46,7 +47,7 @@ func (getter APICInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.Metri } apicInfo.LabelInfo = resourceInfos apicInfo.FilterMetrics = filterMetrics - apicInfo.TTL = time.Now().Add(TTL).Unix() + apicInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return apicInfo.LabelInfo, apicInfo.FilterMetrics } diff --git a/collector/apig.go b/collector/apig.go index af61fd3..f7c6951 100644 --- a/collector/apig.go +++ b/collector/apig.go @@ -45,7 +45,7 @@ func (getter APIGInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricIn } apigInfo.LabelInfo = resourceInfos apigInfo.FilterMetrics = filterMetrics - apigInfo.TTL = time.Now().Add(TTL).Unix() + apigInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return apigInfo.LabelInfo, apigInfo.FilterMetrics } diff --git a/collector/as.go b/collector/as.go index 28f13c0..811f3d8 100644 --- a/collector/as.go +++ b/collector/as.go @@ -34,7 +34,7 @@ func (getter ASInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.MetricI } asInfo.LabelInfo = resourceInfos asInfo.FilterMetrics = filterMetrics - asInfo.TTL = time.Now().Add(TTL).Unix() + asInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return asInfo.LabelInfo, asInfo.FilterMetrics } diff --git a/collector/bms.go b/collector/bms.go index daa275a..982e476 100644 --- a/collector/bms.go +++ b/collector/bms.go @@ -3,8 +3,9 @@ package collector import ( "time" - "github.com/huaweicloud/cloudeye-exporter/logs" cesmodel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) var bmsInfo serversInfo @@ -40,7 +41,7 @@ func (getter BMSInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.Metric } bmsInfo.LabelInfo = resourceInfos bmsInfo.FilterMetrics = filterMetrics - bmsInfo.TTL = time.Now().Add(TTL).Unix() + bmsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return bmsInfo.LabelInfo, bmsInfo.FilterMetrics } diff --git a/collector/cbr.go b/collector/cbr.go index 0603b57..a910776 100644 --- a/collector/cbr.go +++ b/collector/cbr.go @@ -3,12 +3,13 @@ package collector import ( "time" - "github.com/huaweicloud/cloudeye-exporter/logs" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/config" cbr "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/cbr/v1" cbrmodel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/cbr/v1/model" "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) var cbrInfo serversInfo @@ -51,7 +52,7 @@ func (getter CBRInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf cbrInfo.LabelInfo = resourceInfos cbrInfo.FilterMetrics = filterMetrics - cbrInfo.TTL = time.Now().Add(TTL).Unix() + cbrInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return cbrInfo.LabelInfo, cbrInfo.FilterMetrics } diff --git a/collector/cc.go b/collector/cc.go index 18cf3f5..a2ea078 100644 --- a/collector/cc.go +++ b/collector/cc.go @@ -55,7 +55,7 @@ func (getter CCInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.MetricI return ccInfo.LabelInfo, ccInfo.FilterMetrics } ccInfo.LabelInfo, ccInfo.FilterMetrics = buildResourceInfoAndMetrics(metricNames, connections, packages, bandwidths) - ccInfo.TTL = time.Now().Add(TTL).Unix() + ccInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return ccInfo.LabelInfo, ccInfo.FilterMetrics } diff --git a/collector/cdm.go b/collector/cdm.go index 072a0a7..4d20b14 100644 --- a/collector/cdm.go +++ b/collector/cdm.go @@ -41,7 +41,7 @@ func (getter CDMInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.Metric } cdmInfo.LabelInfo = resourceInfos cdmInfo.FilterMetrics = filterMetrics - cdmInfo.TTL = time.Now().Add(TTL).Unix() + cdmInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return cdmInfo.LabelInfo, cdmInfo.FilterMetrics } diff --git a/collector/cfw.go b/collector/cfw.go index 27b8b90..0d6fc8e 100644 --- a/collector/cfw.go +++ b/collector/cfw.go @@ -45,7 +45,7 @@ func (cfw CFWInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.MetricInf metrics := buildSingleDimensionMetrics(metricNames, "SYS.CFW", "fw_instance_id", server.ID) filterMetrics = append(filterMetrics, metrics...) info := labelInfo{ - Name: []string{"name", "ep_id"}, + Name: []string{"name", "epId"}, Value: []string{server.Name, server.EpId}, } keys, values := getTags(server.Tags) @@ -56,7 +56,7 @@ func (cfw CFWInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.MetricInf cfwServerInfo.LabelInfo = resourceInfos cfwServerInfo.FilterMetrics = filterMetrics - cfwServerInfo.TTL = time.Now().Add(TTL).Unix() + cfwServerInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return cfwServerInfo.LabelInfo, cfwServerInfo.FilterMetrics } diff --git a/collector/collector.go b/collector/collector.go index c6ed8d5..33205da 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -59,7 +59,7 @@ func (exporter *BaseHuaweiCloudExporter) listMetrics(namespace string) ([]model. allResourcesInfo, metrics := exporter.listAllResources(namespace) logs.Logger.Debugf("[%s] Resource number of %s: %d", exporter.txnKey, namespace, len(allResourcesInfo)) - if len(metrics) > 0 { + if len(metrics) > 0 && CloudConf.Global.EpIds != "" { return metrics, allResourcesInfo } logs.Logger.Debugf("[%s] Start to getAllMetric from CES", exporter.txnKey) diff --git a/collector/config.go b/collector/config.go index 1736447..2465245 100644 --- a/collector/config.go +++ b/collector/config.go @@ -4,6 +4,9 @@ import ( "errors" "io/ioutil" "net/http" + "path/filepath" + "regexp" + "strings" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/global" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/config" @@ -27,12 +30,14 @@ type CloudAuth struct { } type Global struct { - Port string `yaml:"port"` - Prefix string `yaml:"prefix"` - MetricPath string `yaml:"metric_path"` - EpsInfoPath string `yaml:"eps_path"` - MaxRoutines int `yaml:"max_routines"` - ScrapeBatchSize int `yaml:"scrape_batch_size"` + Port string `yaml:"port"` + Prefix string `yaml:"prefix"` + MetricPath string `yaml:"metric_path"` + EpsInfoPath string `yaml:"eps_path"` + MaxRoutines int `yaml:"max_routines"` + ScrapeBatchSize int `yaml:"scrape_batch_size"` + ResourceSyncIntervalMinutes int `yaml:"resource_sync_interval_minutes"` + EpIds string `yaml:"ep_ids"` } type CloudConfig struct { @@ -46,7 +51,12 @@ var TmpAK string var TmpSK string func InitCloudConf(file string) error { - data, err := ioutil.ReadFile(file) + realPath, err := NormalizePath(file) + if err != nil { + return err + } + + data, err := ioutil.ReadFile(realPath) if err != nil { return err } @@ -67,6 +77,19 @@ func InitCloudConf(file string) error { return err } +func NormalizePath(path string) (string, error) { + relPath, err := filepath.Abs(path) // 对文件路径进行标准化 + if err != nil { + return "", err + } + relPath = strings.Replace(relPath, "\\", "/", -1) + match, err := regexp.MatchString("[!;<>&|$\n`\\\\]", relPath) + if match || err != nil { + return "", errors.New("match path error") + } + return relPath, nil +} + func SetDefaultConfigValues(config *CloudConfig) { if config.Global.Port == "" { config.Global.Port = ":8087" @@ -91,6 +114,10 @@ func SetDefaultConfigValues(config *CloudConfig) { if config.Global.ScrapeBatchSize == 0 { config.Global.ScrapeBatchSize = 300 } + + if config.Global.ResourceSyncIntervalMinutes <= 0 { + config.Global.ResourceSyncIntervalMinutes = 180 + } } type MetricConf struct { diff --git a/collector/config_test.go b/collector/config_test.go index 088c262..e0184d5 100644 --- a/collector/config_test.go +++ b/collector/config_test.go @@ -2,7 +2,9 @@ package collector import ( "errors" + "io/ioutil" "os" + "path/filepath" "testing" "github.com/agiledragon/gomonkey/v2" @@ -112,3 +114,61 @@ func TestInitEndpointConfig(t *testing.T) { }) } } + +func TestSetDefaultConfigValues(t *testing.T) { + config := &CloudConfig{} + SetDefaultConfigValues(config) + assert.NotEqual(t, 0, config.Global.ResourceSyncIntervalMinutes) +} + +func TestInitCloudConf(t *testing.T) { + testCases := []struct { + name string + patches func() *gomonkey.Patches + expect func(t *testing.T, err error) + }{ + { + "NormalizeGetAbsolutePathError", + func() *gomonkey.Patches { + patches := gomonkey.NewPatches() + patches.ApplyFuncReturn(filepath.Abs, "", errors.New("get absolute path error")) + return patches + }, + func(t *testing.T, err error) { + assert.NotNil(t, err) + }, + }, + { + "PathPatternError", + func() *gomonkey.Patches { + patches := gomonkey.NewPatches() + patches.ApplyFuncReturn(filepath.Abs, "/usr/local/\n/cloudeye-exporter/clouds.conf", nil) + return patches + }, + func(t *testing.T, err error) { + assert.NotNil(t, err) + }, + }, + { + "NormalizeSuccessReadError", + func() *gomonkey.Patches { + patches := gomonkey.NewPatches() + patches.ApplyFuncReturn(filepath.Abs, "/usr/local/cloudeye-exporter/clouds.conf", nil) + patches.ApplyFuncReturn(ioutil.ReadFile, nil, errors.New("read file content error")) + return patches + }, + func(t *testing.T, err error) { + assert.NotNil(t, err) + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + patches := testCase.patches() + defer patches.Reset() + err := InitCloudConf("/usr/local/cloudeye-exporter/clouds.conf") + testCase.expect(t, err) + }) + } +} diff --git a/collector/dayu.go b/collector/dayu.go index 88e9723..0849b1f 100644 --- a/collector/dayu.go +++ b/collector/dayu.go @@ -45,7 +45,7 @@ func (getter DayuInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricIn dayuInfo.LabelInfo = resourceInfos dayuInfo.FilterMetrics = filterMetrics - dayuInfo.TTL = time.Now().Add(TTL).Unix() + dayuInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return dayuInfo.LabelInfo, dayuInfo.FilterMetrics } diff --git a/collector/dbss.go b/collector/dbss.go index fef7571..3c2bf8c 100644 --- a/collector/dbss.go +++ b/collector/dbss.go @@ -44,7 +44,7 @@ func (getter DBSSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricIn dbssInfo.LabelInfo = resourceInfos dbssInfo.FilterMetrics = filterMetrics - dbssInfo.TTL = time.Now().Add(TTL).Unix() + dbssInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return dbssInfo.LabelInfo, dbssInfo.FilterMetrics } diff --git a/collector/dcaas.go b/collector/dcaas.go index 79b5db4..b5b5146 100644 --- a/collector/dcaas.go +++ b/collector/dcaas.go @@ -57,7 +57,7 @@ func (getter DCAASInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricI dcaasInfo.LabelInfo = resourceInfos dcaasInfo.FilterMetrics = filterMetrics - dcaasInfo.TTL = time.Now().Add(TTL).Unix() + dcaasInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return dcaasInfo.LabelInfo, dcaasInfo.FilterMetrics } diff --git a/collector/dcs.go b/collector/dcs.go index 40c5d97..df1e0eb 100644 --- a/collector/dcs.go +++ b/collector/dcs.go @@ -37,7 +37,7 @@ func (getter DCSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf dcsInfo.LabelInfo = resourceInfos dcsInfo.FilterMetrics = filterMetrics - dcsInfo.TTL = time.Now().Add(TTL).Unix() + dcsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return dcsInfo.LabelInfo, dcsInfo.FilterMetrics } diff --git a/collector/ddms.go b/collector/ddms.go index 1e2f445..ece99cb 100644 --- a/collector/ddms.go +++ b/collector/ddms.go @@ -52,7 +52,7 @@ func (getter DDMSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricIn } ddmsInfo.LabelInfo = resourceInfos ddmsInfo.FilterMetrics = filterMetrics - ddmsInfo.TTL = time.Now().Add(TTL).Unix() + ddmsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return ddmsInfo.LabelInfo, ddmsInfo.FilterMetrics } diff --git a/collector/ddos.go b/collector/ddos.go index a078280..b09586c 100644 --- a/collector/ddos.go +++ b/collector/ddos.go @@ -40,7 +40,7 @@ func (getter DDOSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricIn } ddosInfo.LabelInfo = resourceInfos ddosInfo.FilterMetrics = filterMetrics - ddosInfo.TTL = time.Now().Add(TTL).Unix() + ddosInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return ddosInfo.LabelInfo, ddosInfo.FilterMetrics } diff --git a/collector/dds.go b/collector/dds.go index 43f3f64..505cbd7 100644 --- a/collector/dds.go +++ b/collector/dds.go @@ -3,12 +3,13 @@ package collector import ( "time" - "github.com/huaweicloud/cloudeye-exporter/logs" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/config" "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" dds "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/dds/v3" ddsmodel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/dds/v3/model" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) type DdsInstanceInfo struct { @@ -71,7 +72,7 @@ func (getter DDSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf } ddsInfo.LabelInfo = resourceInfos ddsInfo.FilterMetrics = filterMetrics - ddsInfo.TTL = time.Now().Add(TTL).Unix() + ddsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return ddsInfo.LabelInfo, ddsInfo.FilterMetrics } diff --git a/collector/dli.go b/collector/dli.go index a53c3c0..6ed5df7 100644 --- a/collector/dli.go +++ b/collector/dli.go @@ -5,8 +5,9 @@ import ( "fmt" "time" - "github.com/huaweicloud/cloudeye-exporter/logs" "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) var dliInfo serversInfo @@ -27,13 +28,43 @@ func (getter DLIInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf // flink jobs buildFlinkJobsInfo(sysConfigMap, &filterMetrics, resourceInfos) + // elastic resource pool + buildElasticPool(sysConfigMap, &filterMetrics, resourceInfos) + dliInfo.LabelInfo = resourceInfos dliInfo.FilterMetrics = filterMetrics - dliInfo.TTL = time.Now().Add(TTL).Unix() + dliInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return dliInfo.LabelInfo, dliInfo.FilterMetrics } +func buildElasticPool(configMap map[string][]string, filterMetrics *[]model.MetricInfoList, infos map[string]labelInfo) { + elasticPoolsMetricNames, ok := configMap["elastic_resource_pool_id"] + if !ok { + logs.Logger.Warnf("metric config is empty of elastic_resource_pool_id") + return + } + + elasticPoolRes, err := getElasticPoolFromDLI() + if err != nil { + logs.Logger.Errorf("Get all elastic pool error: %s", err.Error()) + return + } + + for _, pool := range elasticPoolRes { + metrics := buildSingleDimensionMetrics(elasticPoolsMetricNames, "SYS.DLI", "elastic_resource_pool_id", pool.ID) + *filterMetrics = append(*filterMetrics, metrics...) + info := labelInfo{ + Name: []string{"name", "epId"}, + Value: []string{pool.Name, pool.EpId}, + } + keys, values := getTags(pool.Tags) + info.Name = append(info.Name, keys...) + info.Value = append(info.Value, values...) + infos[GetResourceKeyFromMetricInfo(metrics[0])] = info + } +} + func buildQueuesInfo(sysConfigMap map[string][]string, filterMetrics *[]model.MetricInfoList, resourceInfos map[string]labelInfo) { queueMetricNames, ok := sysConfigMap["queue_id"] if !ok { @@ -85,7 +116,25 @@ func buildFlinkJobsInfo(sysConfigMap map[string][]string, filterMetrics *[]model } func getQueuesFromRMS() ([]ResourceBaseInfo, error) { - return getResourcesBaseInfoFromRMS("dli", "queues") + resp, err := listResources("dli", "queues") + if err != nil { + logs.Logger.Errorf("Failed to list resource of %s.%s, error: %s", "dli", "queues", err.Error()) + return nil, err + } + services := make([]ResourceBaseInfo, len(resp)) + for index, resource := range resp { + var queueProperties DLIQueueProperties + err := fmtResourceProperties(resource.Properties, &queueProperties) + if err != nil { + logs.Logger.Errorf("Fmt dli properties error: %s", err.Error()) + continue + } + services[index].ID = fmt.Sprintf("%d", queueProperties.QueueId) + services[index].Name = *resource.Name + services[index].EpId = *resource.EpId + services[index].Tags = resource.Tags + } + return services, nil } type ListFlinkJobsRequest struct { @@ -123,6 +172,24 @@ type FlinkJobsInfo struct { JobType string } +type ElasticPool struct { + EpID string `json:"enterprise_project_id"` + ID int `json:"id"` + PoolName string `json:"elastic_resource_pool_name"` +} + +type ElasticPoolResponse struct { + IsSuccess bool `json:"is_success"` + Message string `json:"message"` + Count int `json:"count"` + ElasticPools []ElasticPool `json:"elastic_resource_pools"` + HttpStatusCode int `json:"-"` +} + +type DLIQueueProperties struct { + QueueId int `json:"queue_id"` +} + func getAllFlinkJobsInfo() ([]FlinkJobsInfo, error) { var jobs []FlinkJobsInfo limit := int32(100) @@ -151,3 +218,34 @@ func getAllFlinkJobsInfo() ([]FlinkJobsInfo, error) { } return jobs, nil } + +func getElasticPoolFromDLI() ([]ResourceBaseInfo, error) { + var jobs []ResourceBaseInfo + limit := int32(100) + offset := int32(0) + request := &ListFlinkJobsRequest{Limit: &limit, Offset: &offset} + requestDef := genDefaultReqDefWithOffsetAndLimit("/v3/{project_id}/elastic-resource-pools", new(ElasticPoolResponse)) + logs.Logger.Infof("get all pools req is %s", requestDef) + for { + resp, err := getHcClient(getEndpoint("dli", "v3")).Sync(request, requestDef) + logs.Logger.Infof("get all pools finish %s", resp) + if err != nil { + return nil, err + } + poolsInfo, ok := resp.(*ElasticPoolResponse) + if !ok { + return nil, errors.New("resp type is not ElasticPoolResponse") + } + if len(poolsInfo.ElasticPools) == 0 { + break + } + for _, pool := range poolsInfo.ElasticPools { + logs.Logger.Infof("current pool is %s", pool) + jobs = append(jobs, ResourceBaseInfo{ + ID: fmt.Sprintf("%d", pool.ID), Name: pool.PoolName, EpId: pool.EpID, + }) + } + *request.Offset += limit + } + return jobs, nil +} diff --git a/collector/dli_test.go b/collector/dli_test.go new file mode 100644 index 0000000..d051567 --- /dev/null +++ b/collector/dli_test.go @@ -0,0 +1,96 @@ +package collector + +import ( + "errors" + "testing" + + "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/def" + "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" + rmsModel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/rms/v1/model" + "github.com/stretchr/testify/assert" + + "github.com/huaweicloud/cloudeye-exporter/logs" +) + +func TestBuildElasticPoolWhenMapIsEmpty(t *testing.T) { + patches := getPatches() + defer patches.Reset() + logs.InitLog() + resourceInfos := map[string]labelInfo{} + filterMetrics := make([]model.MetricInfoList, 0) + sysConfigMap := getMetricConfigMap("SYS.DLI") + buildElasticPool(sysConfigMap, &filterMetrics, resourceInfos) + assert.Nil(t, sysConfigMap) +} + +func TestBuildElasticPoolWhenGetError(t *testing.T) { + patches := getPatches() + defer patches.Reset() + logs.InitLog() + resourceInfos := map[string]labelInfo{} + filterMetrics := make([]model.MetricInfoList, 0) + sysConfigMap := map[string][]string{} + + metricArr := make([]string, 0, 0) + metricArr = append(metricArr, "elastic_resource_pool_cpu_usage") + sysConfigMap["elastic_resource_pool_id"] = metricArr + buildElasticPool(sysConfigMap, &filterMetrics, resourceInfos) + assert.Equal(t, 0, len(filterMetrics)) +} + +func TestBuildElasticPool(t *testing.T) { + patches := getPatches() + defer patches.Reset() + logs.InitLog() + resourceInfos := map[string]labelInfo{} + filterMetrics := make([]model.MetricInfoList, 0) + sysConfigMap := map[string][]string{} + metricArr := make([]string, 0, 0) + metricArr = append(metricArr, "elastic_resource_pool_cpu_usage") + sysConfigMap["elastic_resource_pool_id"] = metricArr + + respPage1 := ElasticPoolResponse{ + HttpStatusCode: 200, + ElasticPools: []ElasticPool{ + {ID: 442, EpID: "0", PoolName: "Max"}, + }, + } + respPage2 := ElasticPoolResponse{ + HttpStatusCode: 200, + ElasticPools: []ElasticPool{}, + } + patches.ApplyMethodFunc(getHcClient(getEndpoint("dli", "v3")), "Sync", func(req interface{}, reqDef *def.HttpRequestDef) (interface{}, error) { + request, ok := req.(*ListFlinkJobsRequest) + if !ok { + return nil, errors.New("test error") + } + if *request.Offset == 0 { + return &respPage1, nil + } + return &respPage2, nil + }) + + buildElasticPool(sysConfigMap, &filterMetrics, resourceInfos) + assert.Equal(t, 1, len(filterMetrics)) +} + +func TestGetQueuesFromRMS(t *testing.T) { + patches := getPatches() + defer patches.Reset() + logs.InitLog() + id1 := "123123123" + name1 := "test_name" + response := []rmsModel.ResourceEntity{ + { + Id: &id1, + Name: &name1, + Properties: map[string]interface{}{ + "queue_id": "2222", + }, + }, + } + patches.ApplyFuncReturn(listResources, response, nil) + resources, err := getQueuesFromRMS() + assert.Nil(t, err) + assert.Equal(t, 1, len(resources)) +} diff --git a/collector/dms.go b/collector/dms.go index a387a25..6ca58b7 100644 --- a/collector/dms.go +++ b/collector/dms.go @@ -3,13 +3,8 @@ package collector import ( "time" - "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic" - "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/config" - "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" - rocketmq "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/rocketmq/v2" - rocketmqmodel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/rocketmq/v2/model" - "github.com/huaweicloud/cloudeye-exporter/logs" + "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" ) type DmsInstanceInfo struct { @@ -26,7 +21,7 @@ func (getter DMSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf defer dmsInfo.Unlock() if dmsInfo.LabelInfo == nil { dmsInfo.LabelInfo, dmsInfo.FilterMetrics = getDMSResourceAndMetrics() - dmsInfo.TTL = time.Now().Add(TTL).Unix() + dmsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } if time.Now().Unix() > dmsInfo.TTL { go func() { @@ -35,7 +30,7 @@ func (getter DMSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf defer dmsInfo.Unlock() dmsInfo.LabelInfo = label dmsInfo.FilterMetrics = metrics - dmsInfo.TTL = time.Now().Add(TTL).Unix() + dmsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() }() } return dmsInfo.LabelInfo, dmsInfo.FilterMetrics @@ -86,27 +81,3 @@ func getDMSInstanceFromRMS() []ResourceBaseInfo { return instances } - -func getRocketMQInstances() ([]ResourceBaseInfo, error) { - request := &rocketmqmodel.ListInstancesRequest{} - response, err := getRocketMQClient().ListInstances(request) - if err != nil { - logs.Logger.Errorf("Get all RocketMQ instances : %s", err.Error()) - return nil, err - } - - instances := make([]ResourceBaseInfo, len(*response.Instances)) - for index, instance := range *response.Instances { - instances[index].ID = *instance.InstanceId - instances[index].Name = *instance.Name - instances[index].Tags = fmtTags(instance.Tags) - } - return instances, nil -} - -func getRocketMQClient() *rocketmq.RocketMQClient { - return rocketmq.NewRocketMQClient(rocketmq.RocketMQClientBuilder().WithCredential( - basic.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).WithProjectId(conf.ProjectID).Build()). - WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(true)). - WithEndpoint(getEndpoint("dms", "v2")).Build()) -} diff --git a/collector/drs.go b/collector/drs.go index 13d4201..68ea9ed 100644 --- a/collector/drs.go +++ b/collector/drs.go @@ -4,9 +4,10 @@ import ( "fmt" "time" - "github.com/huaweicloud/cloudeye-exporter/logs" "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" rmsmodel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/rms/v1/model" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) type DrsInstanceInfo struct { @@ -50,7 +51,7 @@ func (getter DRSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf drsInfo.LabelInfo = resourceInfos drsInfo.FilterMetrics = filterMetrics - drsInfo.TTL = time.Now().Add(TTL).Unix() + drsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return drsInfo.LabelInfo, drsInfo.FilterMetrics } diff --git a/collector/dws.go b/collector/dws.go index 82c83b0..ffb502c 100644 --- a/collector/dws.go +++ b/collector/dws.go @@ -1,7 +1,6 @@ package collector import ( - "github.com/huaweicloud/cloudeye-exporter/logs" "time" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic" @@ -9,6 +8,8 @@ import ( "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" dws "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/dws/v2" dwsmodel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/dws/v2/model" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) var dwsInfo serversInfo @@ -45,7 +46,7 @@ func (getter DWSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf } dwsInfo.LabelInfo = resourceInfos dwsInfo.FilterMetrics = filterMetrics - dwsInfo.TTL = time.Now().Add(TTL).Unix() + dwsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return dwsInfo.LabelInfo, dwsInfo.FilterMetrics } diff --git a/collector/ecs.go b/collector/ecs.go index 9c241bc..e4eb11c 100644 --- a/collector/ecs.go +++ b/collector/ecs.go @@ -57,7 +57,7 @@ func (getter ECSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf ecsInfo.LabelInfo = resourceInfos ecsInfo.FilterMetrics = filterMetrics - ecsInfo.TTL = time.Now().Add(TTL).Unix() + ecsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return ecsInfo.LabelInfo, ecsInfo.FilterMetrics } @@ -164,7 +164,7 @@ func (getter AGTECSInfo) GetResourceInfo() (map[string]labelInfo, []model.Metric if agtEcsInfo.LabelInfo == nil { agtEcsInfo.FilterMetrics = getECSAGTMetrics() agtEcsInfo.LabelInfo = ecsInfo.LabelInfo - agtEcsInfo.TTL = time.Now().Add(TTL).Unix() + agtEcsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } if time.Now().Unix() > agtEcsInfo.TTL { go func() { @@ -173,7 +173,7 @@ func (getter AGTECSInfo) GetResourceInfo() (map[string]labelInfo, []model.Metric defer agtEcsInfo.Unlock() agtEcsInfo.FilterMetrics = metrics agtEcsInfo.LabelInfo = ecsInfo.LabelInfo - agtEcsInfo.TTL = time.Now().Add(TTL).Unix() + agtEcsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() }() } return agtEcsInfo.LabelInfo, agtEcsInfo.FilterMetrics diff --git a/collector/ecs_test.go b/collector/ecs_test.go new file mode 100644 index 0000000..7ba2bf1 --- /dev/null +++ b/collector/ecs_test.go @@ -0,0 +1,61 @@ +package collector + +import ( + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" + "github.com/stretchr/testify/assert" +) + +func TestECSInfo_GetResourceInfo(t *testing.T) { + patches := + gomonkey.ApplyFuncReturn(getAllServer, []EcsInstancesInfo{ + { + ResourceBaseInfo: ResourceBaseInfo{ + ID: "c53dbd97-c4ae-4ef7-8d12-fae2d3d38a80", + Name: "nodelete-DRS-autotest-datamode1-test2", + Tags: map[string]string{"wukong": "000"}, + EpId: "0", + }, + IP: "192.168.20.53,100.93.4.203", + }, + }, nil) + patches.ApplyFuncReturn(getAllServerFromRMS, []EcsInstancesInfo{ + { + ResourceBaseInfo: ResourceBaseInfo{ + ID: "c53dbd97-c4ae-4ef7-8d12-fae2d3d38a80", + Name: "nodelete-DRS-autotest-datamode1-test2", + Tags: map[string]string{"wukong": "000"}, + EpId: "0", + }, + IP: "192.168.20.53,100.93.4.203", + }, + }, nil) + patches.ApplyFunc(loadAgentDimensions, func(instanceID string) {}) + defer patches.Reset() + ecsInfo1 := ECSInfo{} + _, filterMetrics := ecsInfo1.GetResourceInfo() + assert.NotNil(t, filterMetrics) +} + +func TestAGTECSInfo_GetResourceInfo(t *testing.T) { + unit := "%" + namespace := "AGT.ECS" + metricInfosList := model.MetricInfoList{ + Unit: unit, + Namespace: namespace, + MetricName: "cpu_usage", + Dimensions: []model.MetricsDimension{ + { + Name: "instance_id", + Value: "9234ad9f-87a5-49a9-b6ec-11ecde1d943b", + }, + }, + } + patches := gomonkey.ApplyFuncReturn(getECSAGTMetrics, []model.MetricInfoList{metricInfosList}) + defer patches.Reset() + agtEcsInfo1 := AGTECSInfo{} + _, filterMetrics := agtEcsInfo1.GetResourceInfo() + assert.NotNil(t, filterMetrics) +} diff --git a/collector/efs.go b/collector/efs.go index af6b670..98ba563 100644 --- a/collector/efs.go +++ b/collector/efs.go @@ -3,8 +3,9 @@ package collector import ( "time" - "github.com/huaweicloud/cloudeye-exporter/logs" "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) type EfsInstanceInfo struct { @@ -48,7 +49,7 @@ func (getter EFSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf efsInfo.LabelInfo = resourceInfos efsInfo.FilterMetrics = filterMetrics - efsInfo.TTL = time.Now().Add(TTL).Unix() + efsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return efsInfo.LabelInfo, efsInfo.FilterMetrics } diff --git a/collector/elb.go b/collector/elb.go index 6026577..e87e909 100644 --- a/collector/elb.go +++ b/collector/elb.go @@ -121,7 +121,7 @@ func (getter ELBInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.Metric elbInfo.LabelInfo = resourceInfos elbInfo.FilterMetrics = filterMetrics - elbInfo.TTL = time.Now().Add(TTL).Unix() + elbInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return elbInfo.LabelInfo, elbInfo.FilterMetrics } diff --git a/collector/eps.go b/collector/eps.go index dfba304..2bd67ef 100644 --- a/collector/eps.go +++ b/collector/eps.go @@ -45,7 +45,7 @@ func GetEPSInfo() (string, error) { return result, err } for _, detail := range epsInfo { - result += fmt.Sprintf("huaweicloud_epinfo{epId=\"%s\",epName=\"%s\"} 1\n", detail.Id, detail.Name) + result += fmt.Sprintf("%s_epinfo{epId=\"%s\",epName=\"%s\"} 1\n", CloudConf.Global.Prefix, detail.Id, detail.Name) } return result, nil } @@ -77,6 +77,6 @@ func listEps() ([]model.EpDetail, error) { *req.Offset += limit } epsInfo.EpDetails = resources - epsInfo.TTL = time.Now().Add(TTL).Unix() + epsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() return epsInfo.EpDetails, nil } diff --git a/collector/eps_test.go b/collector/eps_test.go index aab6d7c..7158e3b 100644 --- a/collector/eps_test.go +++ b/collector/eps_test.go @@ -3,6 +3,9 @@ package collector import ( "testing" + "github.com/agiledragon/gomonkey/v2" + eps "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/eps/v1" + "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/eps/v1/model" "github.com/stretchr/testify/assert" ) @@ -13,3 +16,35 @@ func TestGetEPSClient(t *testing.T) { epsClient := getEPSClient() assert.NotNil(t, epsClient) } + +func TestGetEPSInfo(t *testing.T) { + endpointConfig = map[string]string{ + "eps": "https://eps.myhuaweicloud.com", + } + client := eps.NewEpsClient(getEPSClientBuilder().Build()) + patches := gomonkey.NewPatches() + outputs := []gomonkey.OutputCell{ + { + Values: gomonkey.Params{&model.ListEnterpriseProjectResponse{ + EnterpriseProjects: &[]model.EpDetail{ + { + Id: "0", + Name: "default", + Description: "默认企业项目", + Status: 1, + }, + }, + }, nil}, + }, + { + Values: gomonkey.Params{&model.ListEnterpriseProjectResponse{ + EnterpriseProjects: &[]model.EpDetail{}, + }, nil}, + }, + } + patches.ApplyMethodSeq(client, "ListEnterpriseProject", outputs) + patches.ApplyFuncReturn(eps.NewEpsClient, client) + defer patches.Reset() + _, err := GetEPSInfo() + assert.Nil(t, err) +} diff --git a/collector/es.go b/collector/es.go index 4316253..c86a6e9 100644 --- a/collector/es.go +++ b/collector/es.go @@ -45,7 +45,7 @@ func (getter ESInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInfo buildEsResourceInfo(&filterMetrics, resourceInfos) esInfo.LabelInfo = resourceInfos esInfo.FilterMetrics = filterMetrics - esInfo.TTL = time.Now().Add(TTL).Unix() + esInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return esInfo.LabelInfo, esInfo.FilterMetrics } diff --git a/collector/evs.go b/collector/evs.go index 1b3119b..12572ee 100644 --- a/collector/evs.go +++ b/collector/evs.go @@ -78,7 +78,7 @@ func (getter EVSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf evsInfo.LabelInfo = resourceInfos evsInfo.FilterMetrics = allMetrics - evsInfo.TTL = time.Now().Add(TTL).Unix() + evsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return evsInfo.LabelInfo, evsInfo.FilterMetrics } diff --git a/collector/functiongraph.go b/collector/functiongraph.go index 83277f9..636b607 100644 --- a/collector/functiongraph.go +++ b/collector/functiongraph.go @@ -46,7 +46,7 @@ func (getter FunctionGraphInfo) GetResourceInfo() (map[string]labelInfo, []cesmo } functionGraphInfo.LabelInfo = resourceInfos functionGraphInfo.FilterMetrics = filterMetrics - functionGraphInfo.TTL = time.Now().Add(TTL).Unix() + functionGraphInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return functionGraphInfo.LabelInfo, functionGraphInfo.FilterMetrics } diff --git a/collector/gaussdb.go b/collector/gaussdb.go index 5bb43df..e7e043c 100644 --- a/collector/gaussdb.go +++ b/collector/gaussdb.go @@ -3,9 +3,10 @@ package collector import ( "time" - "github.com/huaweicloud/cloudeye-exporter/logs" "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" rmsModel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/rms/v1/model" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) type GaussdbNodeInfo struct { @@ -54,14 +55,14 @@ func (getter GAUSSDBInfo) GetResourceInfo() (map[string]labelInfo, []model.Metri gaussdbInfo.LabelInfo = resourceInfos gaussdbInfo.FilterMetrics = filterMetrics - gaussdbInfo.TTL = time.Now().Add(TTL).Unix() + gaussdbInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return gaussdbInfo.LabelInfo, gaussdbInfo.FilterMetrics } func getAllGaussdbNodesFromRMS() ([]GaussdbNodeInfo, error) { - resp, err := listResources("gaussdb", "nodes") - instanceResp, err := listResources("gaussdb", "instance") + resp, err := listResources("gaussdbformysql", "nodes") + instanceResp, err := listResources("gaussdbformysql", "instance") instanceMap := make(map[string]rmsModel.ResourceEntity) for _, entity := range instanceResp { diff --git a/collector/gaussdb_test.go b/collector/gaussdb_test.go index 52670cd..f2c6eea 100644 --- a/collector/gaussdb_test.go +++ b/collector/gaussdb_test.go @@ -10,15 +10,16 @@ import ( func TestGaussdbGetResourceInfo(t *testing.T) { sysConfig := map[string][]string{"gaussdb_mysql_instance_id,gaussdb_mysql_node_id": {"gaussdb_mysql114_innodb_bufpool_read_ahead"}} - patches := gomonkey.ApplyFuncReturn(getMetricConfigMap, sysConfig) nodes := mockRmsResource() nodes[0].Properties = map[string]interface{}{ "dimensions": []model.MetricsDimension{ - {Name: "gaussdb_mysql_instance_id", Value: "0001-0001-0000001"}, + {Name: "gaussdb_mysql_instance_id", Value: "0001-0001-000001"}, {Name: "gaussdb_mysql_node_id", Value: "node-0001-0001-0000001"}, }, + "instanceId": "0001-0001-000001", } - patches.ApplyFuncReturn(listResources, mockRmsResource(), nil) + patches := gomonkey.ApplyFuncReturn(getMetricConfigMap, sysConfig) + patches.ApplyFuncReturn(listResources, nodes, nil) defer patches.Reset() var gaussdbgetter GAUSSDBInfo diff --git a/collector/gaussdbv5.go b/collector/gaussdbv5.go index 4c7eba3..82f58b1 100644 --- a/collector/gaussdbv5.go +++ b/collector/gaussdbv5.go @@ -68,7 +68,7 @@ func (getter GAUSSDBV5Info) GetResourceInfo() (map[string]labelInfo, []cesmodel. } gaussdbV5Info.LabelInfo = resourceInfos gaussdbV5Info.FilterMetrics = filterMetrics - gaussdbV5Info.TTL = time.Now().Add(TTL).Unix() + gaussdbV5Info.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return gaussdbV5Info.LabelInfo, gaussdbV5Info.FilterMetrics } diff --git a/collector/ges.go b/collector/ges.go index 5c9a814..31cf088 100644 --- a/collector/ges.go +++ b/collector/ges.go @@ -52,7 +52,7 @@ func (getter GESInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf gesInfo.LabelInfo = resourceInfos gesInfo.FilterMetrics = filterMetrics - gesInfo.TTL = time.Now().Add(TTL).Unix() + gesInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return gesInfo.LabelInfo, gesInfo.FilterMetrics } diff --git a/collector/lakeformation.go b/collector/lakeformation.go index d77a45d..0b8c065 100644 --- a/collector/lakeformation.go +++ b/collector/lakeformation.go @@ -52,7 +52,7 @@ func (getter LakeFormationInfo) GetResourceInfo() (map[string]labelInfo, []model lakeformationInfo.LabelInfo = resourceInfos lakeformationInfo.FilterMetrics = filterMetrics - lakeformationInfo.TTL = time.Now().Add(TTL).Unix() + lakeformationInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return lakeformationInfo.LabelInfo, lakeformationInfo.FilterMetrics } diff --git a/collector/model_arts.go b/collector/model_arts.go index 4014c8f..a5ab3c8 100644 --- a/collector/model_arts.go +++ b/collector/model_arts.go @@ -5,9 +5,10 @@ import ( "net/http" "time" - "github.com/huaweicloud/cloudeye-exporter/logs" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/def" cesmodel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) var modelArtsInfo serversInfo @@ -61,7 +62,7 @@ func (getter ModelArtsInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel. } modelArtsInfo.LabelInfo = resourceInfos modelArtsInfo.FilterMetrics = filterMetrics - modelArtsInfo.TTL = time.Now().Add(TTL).Unix() + modelArtsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return modelArtsInfo.LabelInfo, modelArtsInfo.FilterMetrics } diff --git a/collector/mrs.go b/collector/mrs.go index 826c1b5..187dbf0 100644 --- a/collector/mrs.go +++ b/collector/mrs.go @@ -23,7 +23,7 @@ func (getter MRSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf defer mrsInfo.Unlock() if mrsInfo.LabelInfo == nil { mrsInfo.LabelInfo, mrsInfo.FilterMetrics = getMRSResourceAndMetrics() - mrsInfo.TTL = time.Now().Add(TTL).Unix() + mrsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } if time.Now().Unix() > mrsInfo.TTL { go func() { @@ -32,7 +32,7 @@ func (getter MRSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf defer mrsInfo.Unlock() mrsInfo.LabelInfo = label mrsInfo.FilterMetrics = metrics - mrsInfo.TTL = time.Now().Add(TTL).Unix() + mrsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() }() } return mrsInfo.LabelInfo, mrsInfo.FilterMetrics diff --git a/collector/mrs_test.go b/collector/mrs_test.go new file mode 100644 index 0000000..3417fee --- /dev/null +++ b/collector/mrs_test.go @@ -0,0 +1,23 @@ +package collector + +import ( + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" + "github.com/stretchr/testify/assert" +) + +func TestMRSInfo_GetResourceInfo(t *testing.T) { + now := time.Now().Add(-time.Minute).Unix() + mrsInfo1 := MRSInfo{} + patches := gomonkey.ApplyFuncReturn(getMRSResourceAndMetrics, map[string]labelInfo{}, []model.MetricInfoList{}) + defer patches.Reset() + mrsInfo1.GetResourceInfo() + assert.NotNil(t, mrsInfo.LabelInfo) + + mrsInfo.TTL = now + mrsInfo1.GetResourceInfo() + assert.NotNil(t, mrsInfo.LabelInfo) +} diff --git a/collector/nat.go b/collector/nat.go index 0fb352d..6258414 100644 --- a/collector/nat.go +++ b/collector/nat.go @@ -34,7 +34,7 @@ func (getter NATInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.Metric } natInfo.LabelInfo = resourceInfos natInfo.FilterMetrics = filterMetrics - natInfo.TTL = time.Now().Add(TTL).Unix() + natInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return natInfo.LabelInfo, natInfo.FilterMetrics } diff --git a/collector/nosql.go b/collector/nosql.go index 65154b3..7c57288 100644 --- a/collector/nosql.go +++ b/collector/nosql.go @@ -41,7 +41,7 @@ func (getter NoSQLInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricI } nosqlInfo.LabelInfo = resourceInfos nosqlInfo.FilterMetrics = filterMetrics - nosqlInfo.TTL = time.Now().Add(TTL).Unix() + nosqlInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return nosqlInfo.LabelInfo, nosqlInfo.FilterMetrics } diff --git a/collector/nosql_test.go b/collector/nosql_test.go new file mode 100644 index 0000000..afe1e17 --- /dev/null +++ b/collector/nosql_test.go @@ -0,0 +1,17 @@ +package collector + +import ( + "testing" + + "github.com/agiledragon/gomonkey/v2" + nosqlmodel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/gaussdbfornosql/v3/model" + "github.com/stretchr/testify/assert" +) + +func TestNoSQLInfo_GetResourceInfo(t *testing.T) { + patches := gomonkey.ApplyFuncReturn(getAllNoSQLInstances, []nosqlmodel.ListInstancesResult{}, nil) + defer patches.Reset() + noSQLInfo1 := NoSQLInfo{} + _, filteredMetricInfos := noSQLInfo1.GetResourceInfo() + assert.NotNil(t, filteredMetricInfos) +} diff --git a/collector/rds.go b/collector/rds.go index 4a1953d..67146c8 100644 --- a/collector/rds.go +++ b/collector/rds.go @@ -64,7 +64,7 @@ func (getter RDSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf rdsInfo.LabelInfo = resourceInfos rdsInfo.FilterMetrics = filterMetrics - rdsInfo.TTL = time.Now().Add(TTL).Unix() + rdsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return rdsInfo.LabelInfo, rdsInfo.FilterMetrics } diff --git a/collector/rms.go b/collector/rms.go index c166291..8b2913e 100644 --- a/collector/rms.go +++ b/collector/rms.go @@ -1,11 +1,16 @@ package collector import ( + "errors" + "strings" + http_client "github.com/huaweicloud/huaweicloud-sdk-go-v3/core" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/global" v1 "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/rms/v1" "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/rms/v1/model" "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/rms/v1/region" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) func getRMSClient() *v1.RmsClient { @@ -24,12 +29,31 @@ func getRMSClientBuilder() *http_client.HcHttpClientBuilder { func listResources(provider, resourceType string) ([]model.ResourceEntity, error) { limit := int32(200) + var resources []model.ResourceEntity req := &model.ListResourcesRequest{ Provider: provider, Type: resourceType, RegionId: &conf.Region, Limit: &limit, } + if CloudConf.Global.EpIds != "" { + epIdArr := strings.Split(CloudConf.Global.EpIds, ",") + for _, epID := range epIdArr { + req.EpId = &epID + resourceByEpID, err := getResourcesFromRMS(req) + if err != nil { + logs.Logger.Errorf("Get resources from rms by epID failed, epID is %s, error: %s", epID, err.Error()) + return nil, errors.New("get resources from rms by epID failed") + } + resources = append(resources, resourceByEpID...) + } + return resources, nil + } else { + return getResourcesFromRMS(req) + } +} + +func getResourcesFromRMS(req *model.ListResourcesRequest) ([]model.ResourceEntity, error) { var resources []model.ResourceEntity for { response, err := getRMSClient().ListResources(req) diff --git a/collector/rms_test.go b/collector/rms_test.go index 04d127e..545d926 100644 --- a/collector/rms_test.go +++ b/collector/rms_test.go @@ -3,7 +3,10 @@ package collector import ( "testing" + "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/rms/v1/model" "github.com/stretchr/testify/assert" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) func TestGetRmsClient(t *testing.T) { @@ -13,3 +16,87 @@ func TestGetRmsClient(t *testing.T) { client := getRMSClient() assert.NotNil(t, client) } + +func TestListResources(t *testing.T) { + patches := getPatches() + defer patches.Reset() + logs.InitLog() + + id1 := "123123123" + name1 := "test_name" + currentNum := int32(1) + epID := "0" + response := model.ListResourcesResponse{ + Resources: &[]model.ResourceEntity{ + { + Id: &id1, + Name: &name1, + EpId: &epID, + Properties: map[string]interface{}{ + "queue_id": "2222", + }, + }, + }, + PageInfo: &model.PageInfo{ + CurrentCount: ¤tNum, + }, + } + endpointConfig = map[string]string{ + "rms": "https://rms.myhuaweicloud.com", + } + + patches.ApplyMethodFunc(getRMSClient(), "ListResources", func(request *model.ListResourcesRequest) (*model.ListResourcesResponse, error) { + return &response, nil + }) + + CloudConf.Global.EpIds = "0" + resources, err := listResources("ecs", "cloudservers") + assert.Nil(t, err) + assert.Equal(t, 1, len(resources)) +} + +func TestListAllResourcesFromRMS(t *testing.T) { + patches := getPatches() + defer patches.Reset() + logs.InitLog() + + id1 := "123123123" + name1 := "test_name" + currentNum := int32(1) + epID1 := "0" + epID2 := "xxxxx" + response := model.ListResourcesResponse{ + Resources: &[]model.ResourceEntity{ + { + Id: &id1, + Name: &name1, + EpId: &epID1, + Properties: map[string]interface{}{ + "queue_id": "2222", + }, + }, + { + Id: &id1, + Name: &name1, + EpId: &epID2, + Properties: map[string]interface{}{ + "queue_id": "2222", + }, + }, + }, + PageInfo: &model.PageInfo{ + CurrentCount: ¤tNum, + }, + } + endpointConfig = map[string]string{ + "rms": "https://rms.myhuaweicloud.com", + } + + patches.ApplyMethodFunc(getRMSClient(), "ListResources", func(request *model.ListResourcesRequest) (*model.ListResourcesResponse, error) { + return &response, nil + }) + + resources, err := listResources("ecs", "cloudservers") + assert.Nil(t, err) + assert.Equal(t, 2, len(resources)) +} diff --git a/collector/sfs.go b/collector/sfs.go index 4a451a0..1979ee7 100644 --- a/collector/sfs.go +++ b/collector/sfs.go @@ -4,8 +4,9 @@ import ( "errors" "time" - "github.com/huaweicloud/cloudeye-exporter/logs" "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) var sfsInfo serversInfo @@ -41,7 +42,7 @@ func (getter SFSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf sfsInfo.LabelInfo = resourceInfos sfsInfo.FilterMetrics = filterMetrics - sfsInfo.TTL = time.Now().Add(TTL).Unix() + sfsInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return sfsInfo.LabelInfo, sfsInfo.FilterMetrics } diff --git a/collector/utils.go b/collector/utils.go index 1f3ebca..e840fec 100644 --- a/collector/utils.go +++ b/collector/utils.go @@ -20,7 +20,7 @@ import ( "github.com/huaweicloud/cloudeye-exporter/logs" ) -const TTL = time.Hour * 3 +const MinimumResourceInfoSyncInterval = 10 var tagRegexp *regexp.Regexp @@ -251,3 +251,11 @@ func getResourcesBaseInfoFromRMS(provider, resourceType string) ([]ResourceBaseI } return services, nil } + +func GetResourceInfoExpirationTime() time.Duration { + intervalMinutes := CloudConf.Global.ResourceSyncIntervalMinutes + if intervalMinutes <= MinimumResourceInfoSyncInterval { + return MinimumResourceInfoSyncInterval * time.Minute + } + return time.Duration(intervalMinutes) * time.Minute +} diff --git a/collector/utils_test.go b/collector/utils_test.go index 1ede00e..55d3e4a 100644 --- a/collector/utils_test.go +++ b/collector/utils_test.go @@ -2,6 +2,7 @@ package collector import ( "testing" + "time" "github.com/agiledragon/gomonkey/v2" cbrmodel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/cbr/v1/model" @@ -265,3 +266,13 @@ func TestGetHcClient(t *testing.T) { client := getHcClient("ces.test.huawei.com") assert.Equal(t, true, client != nil) } + +func TestGetResourceInfoExpirationTime(t *testing.T) { + CloudConf.Global.ResourceSyncIntervalMinutes = 180 + expirationTime := GetResourceInfoExpirationTime() + assert.Equal(t, 180*time.Minute, expirationTime) + + CloudConf.Global.ResourceSyncIntervalMinutes = 5 + expirationTime = GetResourceInfoExpirationTime() + assert.Equal(t, 10*time.Minute, expirationTime) +} diff --git a/collector/version.go b/collector/version.go new file mode 100644 index 0000000..21f53f5 --- /dev/null +++ b/collector/version.go @@ -0,0 +1,5 @@ +package collector + +const ( + Version = "v2.0.9" +) diff --git a/collector/vpc.go b/collector/vpc.go index d625a01..193974d 100644 --- a/collector/vpc.go +++ b/collector/vpc.go @@ -50,7 +50,7 @@ func (getter VPCInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf vpcInfo.LabelInfo = resourceInfos vpcInfo.FilterMetrics = filterMetrics - vpcInfo.TTL = time.Now().Add(TTL).Unix() + vpcInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return vpcInfo.LabelInfo, vpcInfo.FilterMetrics } diff --git a/collector/vpcep.go b/collector/vpcep.go index a0699f8..fce96c7 100644 --- a/collector/vpcep.go +++ b/collector/vpcep.go @@ -3,8 +3,9 @@ package collector import ( "time" - "github.com/huaweicloud/cloudeye-exporter/logs" cesmodel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) var vpcEPInfo serversInfo @@ -39,7 +40,7 @@ func (getter VPCEPInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.Metr } vpcEPInfo.LabelInfo = resourceInfos vpcEPInfo.FilterMetrics = filterMetrics - vpcEPInfo.TTL = time.Now().Add(TTL).Unix() + vpcEPInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return vpcEPInfo.LabelInfo, vpcEPInfo.FilterMetrics } diff --git a/collector/vpcep_test.go b/collector/vpcep_test.go new file mode 100644 index 0000000..c260ac2 --- /dev/null +++ b/collector/vpcep_test.go @@ -0,0 +1,16 @@ +package collector + +import ( + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/assert" +) + +func TestVPCEPInfo_GetResourceInfo(t *testing.T) { + patches := gomonkey.ApplyFuncReturn(getVpcEpEndpoints, []VpcEpEndpoint{}, nil) + defer patches.Reset() + vpcEpInfo1 := VPCEPInfo{} + _, filteredMetricInfo := vpcEpInfo1.GetResourceInfo() + assert.NotNil(t, filteredMetricInfo) +} diff --git a/collector/vpn.go b/collector/vpn.go index a432732..b7a575f 100644 --- a/collector/vpn.go +++ b/collector/vpn.go @@ -54,7 +54,7 @@ func (getter VPNInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf buildEVPNSaInfo(&filterMetrics, resourceInfos, evpnSaMap) vpnInfo.LabelInfo = resourceInfos vpnInfo.FilterMetrics = filterMetrics - vpnInfo.TTL = time.Now().Add(TTL).Unix() + vpnInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return vpnInfo.LabelInfo, vpnInfo.FilterMetrics } diff --git a/collector/waf.go b/collector/waf.go index c2642e6..f89d123 100644 --- a/collector/waf.go +++ b/collector/waf.go @@ -4,12 +4,13 @@ import ( "net/http" "time" - "github.com/huaweicloud/cloudeye-exporter/logs" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic" "github.com/huaweicloud/huaweicloud-sdk-go-v3/core/config" "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model" waf "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/waf/v1" wafModel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/waf/v1/model" + + "github.com/huaweicloud/cloudeye-exporter/logs" ) var wafInfo serversInfo @@ -62,7 +63,7 @@ func (getter WAFInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf wafInfo.LabelInfo = resourceInfos wafInfo.FilterMetrics = filterMetrics - wafInfo.TTL = time.Now().Add(TTL).Unix() + wafInfo.TTL = time.Now().Add(GetResourceInfoExpirationTime()).Unix() } return wafInfo.LabelInfo, wafInfo.FilterMetrics } diff --git a/grafana_dashboard/templates/evs_dashboard_template.json b/grafana_dashboard/templates/evs_dashboard_template.json new file mode 100644 index 0000000..1a09296 --- /dev/null +++ b/grafana_dashboard/templates/evs_dashboard_template.json @@ -0,0 +1,1344 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.4.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "【华为云EVS监控指标说明】https://support.huaweicloud.com/usermanual-evs/evs_01_0044.html", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "个" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "huaweicloud_sys_evs_disk_device_queue_length{epId=~\"$EPS\"}", + "instant": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "$EPS:平均队列长度", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms/op" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "huaweicloud_sys_evs_disk_device_io_svctm{epId=~\"$EPS\"}", + "instant": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "$EPS:平均IO服务时长", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "KB/op" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "huaweicloud_sys_evs_disk_device_read_bytes_per_operation{epId=~\"$EPS\"}", + "instant": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "$EPS:平均读操作大小", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms/op" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "huaweicloud_sys_evs_disk_device_write_await{epId=~\"$EPS\"}", + "instant": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "$EPS:平均写操作耗时", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "次" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "huaweicloud_sys_evs_disk_device_io_iops_qos_num{epId=~\"$EPS\"}", + "instant": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "$EPS:带宽达到上限（次数）", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "次" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "huaweicloud_sys_evs_disk_device_io_iops_qos_num{epId=~\"$EPS\"}", + "instant": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "$EPS:带宽达到上限（次数）", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "请求/秒" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "huaweicloud_sys_evs_disk_device_read_requests_rate{epId=~\"$EPS\"}", + "instant": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "$EPS:磁盘读IOPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "请求/秒" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "huaweicloud_sys_evs_disk_device_write_requests_rate{epId=~\"$EPS\"}", + "instant": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "$EPS:磁盘写IOPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Byte/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "huaweicloud_sys_evs_disk_device_read_bytes_rate{epId=~\"$EPS\"}", + "instant": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "$EPS:磁盘读带宽", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Byte/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "huaweicloud_sys_evs_disk_device_write_bytes_rate{epId=~\"$EPS\"}", + "instant": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "$EPS:磁盘写带宽", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "KB/op" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "huaweicloud_sys_evs_disk_device_write_bytes_per_operation{epId=~\"$EPS\"}", + "instant": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "$EPS:平均写操作大小", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms/op" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "huaweicloud_sys_evs_disk_device_read_await{epId=~\"$EPS\"}", + "instant": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "$EPS:平均读操作耗时", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "%" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "huaweicloud_sys_evs_disk_device_io_util{epId=~\"$EPS\"}", + "instant": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "$EPS:平均读写使用率", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "huaweicloud_epinfo", + "hide": 0, + "includeAll": false, + "label": "企业项目", + "multi": false, + "name": "EPS", + "options": [], + "query": { + "qryType": 4, + "query": "huaweicloud_epinfo", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/epName=\"(?[^\"]+)|epId=\"(?[^\"]+)/g", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "云硬盘（EVS）", + "uid": "ddjveunx3focgf", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana_dashboard/use_grafana_template.md b/grafana_dashboard/use_grafana_template.md index 52eb0cf..06a8073 100644 --- a/grafana_dashboard/use_grafana_template.md +++ b/grafana_dashboard/use_grafana_template.md @@ -49,6 +49,7 @@ + [云数据库 GeminiDB-cassandra](templates/nosql_cassandra_dashboard_template.json) + [DDOS高防 DDOS](templates/ddos_dashboard_template.json) + [内容分发网络 CDN](templates/cdn_dashboard_template.json) + + [云硬盘 EVS](templates/evs_dashboard_template.json) 4. 效果展示： >ECS: > ![img.png](pic/ecs.jpg) diff --git a/main.go b/main.go index 30d22ca..f899476 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "net/http" + "os" "strings" "time" @@ -17,6 +18,7 @@ import ( var ( clientConfig = flag.String("config", "./clouds.yml", "Path to the cloud configuration file") securityMod = flag.Bool("s", false, "Get ak sk from command line") + getVersion = flag.Bool("v", false, "Get version from command line") ak, sk string ) @@ -64,8 +66,16 @@ func getAkSkFromCommandLine() { } } +func getVersionFunc() { + if *getVersion { + fmt.Printf("cloudeye-exporter version: %s", collector.Version) + os.Exit(0) + } +} + func main() { flag.Parse() + getVersionFunc() logs.InitLog() getAkSkFromCommandLine() err := collector.InitCloudConf(*clientConfig) diff --git a/metric.yml b/metric.yml index 351af39..be2ffd6 100644 --- a/metric.yml +++ b/metric.yml @@ -1067,6 +1067,15 @@ SYS.DLI: - flink_mem_usage - flink_max_op_latency - flink_max_op_backpressure_level + elastic_resource_pool_id: + - elastic_resource_pool_cpu_usage + - elastic_resource_pool_mem_usage + - elastic_resource_pool_disk_usage + - elastic_resource_pool_disk_max_usage + - elastic_resource_pool_cu_num + - elastic_resource_pool_alloc_cu_num + - elastic_resource_pool_min_cu_num + - elastic_resource_pool_max_cu_num SYS.SFS: dim_metric_name: share_id: