Skip to content

Commit

Permalink
Merge branch 'feat/object-alias-name' of github.com:TencentBlueKing/b…
Browse files Browse the repository at this point in the history 
…k-monitor into deploy/paas3-dev-0116
  • Loading branch information
@yiqiwang-17
yiqiwang-17 committed Jan 16, 2025
2 parents 2cc9b48 + 498591e commit 98d7b65
Show file tree
Hide file tree
Showing 221 changed files with 9,782 additions and 1,754 deletions.
7 changes: 5 additions & 2 deletions bklog/apps/api/modules/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,11 @@ def add_esb_info_before_request_for_bkdata_token(params): # pylint: disable=fun
req = get_request()
skip_check = getattr(req, "skip_check", False)
if settings.BKAPP_IS_BKLOG_API and not skip_check:
# 外部版请求转发已通过日志鉴权逻辑，默认使用超级权限
if getattr(req, "external_user", None):
# 外部版请求转发 / 内嵌日志api页面 已通过日志saas鉴权逻辑，默认使用超级权限
if (
getattr(req, "external_user", None)
or req.headers.get("X-SOURCE-APP-CODE", "") in settings.ESQUERY_WHITE_LIST
):
params = update_bkdata_auth_info(params)
else:
auth_info = EsquerySearchPermissions.get_auth_info(req)
Expand Down
300 changes: 63 additions & 237 deletions bklog/apps/bk_log_admin/handlers/index_set.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,58 +19,38 @@
We undertake not to change the open source license (MIT license) applicable to the current version of
the project delivered to anyone in the future.
"""
from typing import Dict, Any, List

import arrow
from django.conf import settings
import pytz
from django.db.models import Case, CharField, Count, Value, When
from django.db.models.functions import TruncDate
from django.utils.encoding import force_text

from apps.bk_log_admin.constants import (
BK_DATA_CUSTOM_REPORT_USER_INDEX_SET_HISTORY,
OPERATION_PIE_CHOICE_MAP,
BK_DATA_CUSTOM_REPORT_USER_INDEX_SET_HISTORY_FIELD,
DATE_HISTOGRAM_INTERVAL,
)
from apps.bk_log_admin.constants import OPERATION_PIE_CHOICE_MAP
from apps.log_search.models import UserIndexSetSearchHistory
from apps.models import model_to_dict
from apps.utils.drf import DataPageNumberPagination
from apps.utils.local import get_local_param
from apps.utils.log import logger
from apps.utils.lucene import generate_query_string
from bk_monitor.api.client import Client
from config.domains import MONITOR_APIGATEWAY_ROOT


class IndexSetHandler(object):
def __init__(self):
self._client = Client(
bk_app_code=settings.APP_CODE,
bk_app_secret=settings.SECRET_KEY,
monitor_host=MONITOR_APIGATEWAY_ROOT,
report_host=f"{settings.BKMONITOR_CUSTOM_PROXY_IP}/",
bk_username="admin",
)

@property
def data_label(self):
data_label = "{bk_biz_id}_{app_code}_{table}".format(
bk_biz_id=settings.BLUEKING_BK_BIZ_ID,
app_code=settings.APP_CODE.replace("-", "_"),
table=BK_DATA_CUSTOM_REPORT_USER_INDEX_SET_HISTORY
)
return data_label

@property
def table(self):
return f"{self.data_label}.base"

@property
def prometheus_table(self):
return "custom:{bk_biz_id}_{app_code}_{table}:{field}".format(
bk_biz_id=settings.BLUEKING_BK_BIZ_ID,
app_code=settings.APP_CODE.replace("-", "_"),
table=BK_DATA_CUSTOM_REPORT_USER_INDEX_SET_HISTORY,
field=BK_DATA_CUSTOM_REPORT_USER_INDEX_SET_HISTORY_FIELD
@staticmethod
def get_user_index_set_history_objs(index_set_id, start_time, end_time):
"""
获取索引集用户检索记录表数据
:param index_set_id: 索引集id
:param start_time: 起始时间
:param end_time: 结束时间
"""
objs = UserIndexSetSearchHistory.objects.filter(
index_set_id=index_set_id,
search_type="default",
created_at__range=[
start_time.datetime,
end_time.datetime,
],
)
return objs

def get_date_histogram(self, index_set_id, user_search_history_operation_time):
"""
Expand All @@ -79,41 +59,24 @@ def get_date_histogram(self, index_set_id, user_search_history_operation_time):
@param user_search_history_operation_time.start_time {Str} the search begin
@param user_search_history_operation_time.end_time the search end
"""
start_time, end_time = self._get_start_end_time(
user_search_history_operation_time=user_search_history_operation_time
user_index_set_history_objs = self.get_user_index_set_history_objs(
index_set_id,
arrow.get(user_search_history_operation_time["start_time"]),
arrow.get(user_search_history_operation_time["end_time"]),
)
metrics = [
{
"field": BK_DATA_CUSTOM_REPORT_USER_INDEX_SET_HISTORY_FIELD,
"method": "COUNT",
"alias": "a"
}
]
where = [
{
"key": "index_set_id",
"method": "eq",
"value": [
str(index_set_id)
]
}
]
unify_query_data = self.call_unify_query(
start_time=start_time,
end_time=end_time,
metrics=metrics,
where=where,
interval=DATE_HISTOGRAM_INTERVAL
user_index_set_history = (
user_index_set_history_objs.annotate(
day=TruncDate("created_at", tzinfo=pytz.timezone("UTC")),
)
.values("day")
.annotate(count=Count('id'))
)
if not unify_query_data["series"]:
return {"labels": [], "values": []}

daily_label_list = []
daily_data_list = []
# 只有一个series
for data in unify_query_data["series"][0]["datapoints"]:
daily_label_list.append(arrow.get(data[1] / 1000).format())
daily_data_list.append(data[0] if data[0] else 0)
for item in user_index_set_history:
agg_date = arrow.get(item["day"]).format()
daily_label_list.append(agg_date)
daily_data_list.append(item["count"])

return {"labels": daily_label_list, "values": daily_data_list}

Expand All @@ -124,41 +87,17 @@ def get_user_terms(self, index_set_id, user_search_history_operation_time):
@param user_search_history_operation_time.start_time {Str} the search begin
@param user_search_history_operation_time.end_time the search end
"""
start_time, end_time = self._get_start_end_time(
user_search_history_operation_time=user_search_history_operation_time
user_index_set_history_objs = self.get_user_index_set_history_objs(
index_set_id,
arrow.get(user_search_history_operation_time["start_time"]),
arrow.get(user_search_history_operation_time["end_time"]),
)
metrics = [
{
"field": BK_DATA_CUSTOM_REPORT_USER_INDEX_SET_HISTORY_FIELD,
"method": "COUNT",
"alias": "a"
}
]
where = [
{
"key": "index_set_id",
"method": "eq",
"value": [
str(index_set_id)
]
}
]
group_by = ["created_by"]
unify_query_data = self.call_unify_query(
start_time=start_time,
end_time=end_time,
metrics=metrics,
where=where,
group_by=group_by
)
if not unify_query_data["series"]:
return {"labels": [], "values": []}
user_index_set_history = user_index_set_history_objs.values("created_by").annotate(count=Count("id"))
created_by_label_list = []
created_by_data_list = []
for data in unify_query_data["series"]:
created_by_label_list.append(data["dimensions"]["created_by"])
created_by_data_list.append(sum([i[0] for i in data["datapoints"] if i[0]]))

for item in user_index_set_history:
created_by_label_list.append(item["created_by"])
created_by_data_list.append(item["count"])
return {"labels": created_by_label_list, "values": created_by_data_list}

def get_duration_terms(self, index_set_id, user_search_history_operation_time):
Expand All @@ -168,33 +107,29 @@ def get_duration_terms(self, index_set_id, user_search_history_operation_time):
@param user_search_history_operation_time.start_time {Str} the search begin
@param user_search_history_operation_time.end_time the search end
"""
start_time, end_time = self._get_start_end_time(
user_search_history_operation_time=user_search_history_operation_time
user_index_set_history_objs = self.get_user_index_set_history_objs(
index_set_id,
arrow.get(user_search_history_operation_time["start_time"]),
arrow.get(user_search_history_operation_time["end_time"]),
)

# 根据分组范围和对应的标签构建Case表达式
case_expression = Case(
*[
When(duration__gte=item["min"], duration__lt=item["max"], then=Value(force_text(item["label"])))
if item.get("max")
else When(duration__gte=item["min"], then=Value(force_text(item["label"])))
for item in OPERATION_PIE_CHOICE_MAP
],
output_field=CharField(),
)
results = user_index_set_history_objs.annotate(duration_range=case_expression)
grouped_results = results.values("duration_range").annotate(count=Count("id"))
pie_label_list = []
pie_data_list = []
for pie_choice in OPERATION_PIE_CHOICE_MAP:
pie_label_list.append(pie_choice["label"])
if "min" in pie_choice and "max" in pie_choice:
promql = (
f"count({self.prometheus_table}{{index_set_id=\"{index_set_id}\"}} >= {pie_choice['min']} and "
f"{self.prometheus_table}{{index_set_id=\"{index_set_id}\"}} < {pie_choice['max']})"
)
elif "min" in pie_choice:
promql = f"count({self.prometheus_table}{{index_set_id=\"{index_set_id}\"}} >= {pie_choice['min']})"
# "max" in pie_choice
else:
promql = f"count({self.prometheus_table}{{index_set_id=\"{index_set_id}\"}} < {pie_choice['max']})"
unify_query_data = self.call_unify_query_by_promql(
start_time=start_time,
end_time=end_time,
promql=promql
)
if not unify_query_data["series"]:
pie_data_list.append(0)
continue
pie_data_list.append(sum([i[0] for i in unify_query_data["series"][0]["datapoints"] if i[0]]))

for item in grouped_results:
pie_label_list.append(item["duration_range"])
pie_data_list.append(item["count"])
return {"labels": pie_label_list, "values": pie_data_list}

def list_user_set_history(self, start_time, end_time, request, view, index_set_id):
Expand Down Expand Up @@ -226,112 +161,3 @@ def list_user_set_history(self, start_time, end_time, request, view, index_set_i
def build_query_string(history):
history["query_string"] = generate_query_string(history["params"])
return history

@staticmethod
def _get_start_end_time(user_search_history_operation_time):
start_time = int(user_search_history_operation_time["start_time"])
end_time = int(user_search_history_operation_time["end_time"])
return start_time, end_time

def call_unify_query(
self,
start_time: int,
end_time: int,
metrics: List[Dict[str, Any]] = None,
where: List[Dict[str, Any]] = None,
group_by: List = None,
interval: int = None
):
"""
以通用的形式查询unify_query
"""
if not metrics:
metrics = []
if not where:
where = []
if not group_by:
group_by = []
if not interval:
interval = "auto"
params = {
"down_sample_range": "15m",
"step": interval,
"format": "time_series",
"type": "range",
"start_time": start_time,
"end_time": end_time,
"expression": "a",
"display": True,
"query_configs": [
{
"data_label": self.data_label,
"data_source_label": "custom",
"data_type_label": "time_series",
"metrics": metrics,
"table": self.table,
"group_by": group_by,
"display": True,
"where": where,
"interval": interval,
"interval_unit": "s",
"time_field": "time",
"filter_dict": {},
"functions": []
}
],
"target": [],
"bk_biz_id": str(settings.BLUEKING_BK_BIZ_ID)
}
try:
return self._client.unify_query(params)
except Exception as e:
logger.error(f"unify_query error, params: {params}, error: {e}")
return {
"metrics": [],
"series": [],
}

def call_unify_query_by_promql(
self,
start_time: int,
end_time: int,
promql: str,
interval: int = None
):
"""
以promql的形式查询unify_query
"""
if not interval:
interval = "auto"
params = {
"down_sample_range": "15m",
"step": interval,
"format": "time_series",
"type": "range",
"start_time": start_time,
"end_time": end_time,
"expression": "a",
"display": True,
"query_configs": [
{
"data_label": self.data_label,
"data_source_label": "prometheus",
"data_type_label": "time_series",
"display": True,
"interval": interval,
"interval_unit": "s",
"time_field": "time",
"promql": promql
}
],
"target": [],
"bk_biz_id": str(settings.BLUEKING_BK_BIZ_ID)
}
try:
return self._client.unify_query(params)
except Exception as e:
logger.error(f"unify_query_by_promql error, params: {params}, error: {e}")
return {
"metrics": [],
"series": [],
}
Loading

0 comments on commit 98d7b65

Please sign in to comment.