Merge branch 'veops:master' into master

.gitignore

@@ -78,4 +78,3 @@ cmdb-ui/npm-debug.log*
 cmdb-ui/yarn-debug.log*
 cmdb-ui/yarn-error.log*
 cmdb-ui/package-lock.json
-start.sh

cmdb-api/.ruff.toml

@@ -0,0 +1,79 @@
+line-length = 120
+cache-dir = ".ruff_cache"
+target-version = "py310"
+unsafe-fixes = true
+show-fixes = true
+
+[lint]
+select = [
+    "E",
+    "F",
+    "I",
+    "TCH",
+    # W
+    "W505",
+    # PT
+    "PT018",
+    # SIM
+    "SIM101",
+    "SIM114",
+    # PGH
+    "PGH004",
+    # PL
+    "PLE1142",
+    # RUF
+    "RUF100",
+    # UP
+    "UP007"
+]
+preview = true
+ignore = ["FURB101"]
+
+[lint.flake8-pytest-style]
+mark-parentheses = false
+parametrize-names-type = "list"
+parametrize-values-row-type = "list"
+parametrize-values-type = "tuple"
+
+[lint.flake8-unused-arguments]
+ignore-variadic-names = true
+
+[lint.isort]
+lines-between-types = 1
+order-by-type = true
+
+[lint.per-file-ignores]
+"**/api/v1/*.py" = ["TCH"]
+"**/model/*.py" = ["TCH003"]
+"**/models/__init__.py" = ["F401", "F403"]
+"**/tests/*.py" = ["E402"]
+"celery_worker.py" = ["F401"]
+"api/views/entry.py" = ["I001"]
+"migrations/*.py" = ["I001", "E402"]
+"*.py" = ["I001"]
+"api/views/common_setting/department.py" = ["F841"]
+"api/lib/common_setting/upload_file.py" = ["F841"]
+"api/lib/common_setting/acl.py" = ["F841"]
+"**/__init__.py" = ["F822"]
+"api/tasks/*.py" = ["E722"]
+"api/views/cmdb/*.py" = ["E722"]
+"api/views/acl/*.py" = ["E722"]
+"api/lib/secrets/*.py" = ["E722", "F841"]
+"api/lib/utils.py" = ["E722", "E731"]
+"api/lib/perm/authentication/cas/*" = ["E113", "F841"]
+"api/lib/perm/acl/*" = ["E722"]
+"api/lib/*" = ["E721", "F722"]
+"api/lib/cmdb/*" = ["F722", "E722"]
+"api/lib/cmdb/search/ci/es/search.py" = ["F841", "SIM114"]
+"api/lib/cmdb/search/ci/db/search.py" = ["F841"]
+"api/lib/cmdb/value.py" = ["F841"]
+"api/lib/cmdb/history.py" = ["E501"]
+"api/commands/common.py" = ["E722"]
+"api/commands/click_cmdb.py" = ["E722"]
+"api/lib/perm/auth.py" = ["SIM114"]
+
+[format]
+preview = true
+quote-style = "single"
+docstring-code-format = true
+skip-magic-trailing-comma = false

cmdb-api/api/commands/click_cmdb.py

@@ -228,6 +228,12 @@ def cmdb_trigger():
     """
     from api.lib.cmdb.ci import CITriggerManager
 
+    current_app.test_request_context().push()
+    if not UserCache.get('worker'):
+        from api.lib.perm.acl.user import UserCRUD
+        UserCRUD.add(username='worker', password=uuid.uuid4().hex, email='[email protected]')
+    login_user(UserCache.get('worker'))
+
     current_day = datetime.datetime.today().strftime("%Y-%m-%d")
     trigger2cis = dict()
     trigger2completed = dict()
@@ -256,10 +262,10 @@ def cmdb_trigger():
                         trigger2cis[trigger.id] = (trigger, ready_cis)
                     else:
                         cur = trigger2cis[trigger.id]
-                        cur_ci_ids = {i.ci_id for i in cur[1]}
+                        cur_ci_ids = {_ci.ci_id for _ci in cur[1]}
                         trigger2cis[trigger.id] = (
-                            trigger, cur[1] + [i for i in ready_cis if i.ci_id not in cur_ci_ids
-                                               and i.ci_id not in trigger2completed.get(trigger.id, {})])
+                            trigger, cur[1] + [_ci for _ci in ready_cis if _ci.ci_id not in cur_ci_ids
+                                               and _ci.ci_id not in trigger2completed.get(trigger.id, {})])
 
             for tid in trigger2cis:
                 trigger, cis = trigger2cis[tid]
@@ -346,7 +352,7 @@ def cmdb_inner_secrets_init(address):
     if valid_address(address):
         token = current_app.config.get("INNER_TRIGGER_TOKEN", "") if not token else token
         if not token:
-            token = click.prompt(f'Enter root token', hide_input=True, confirmation_prompt=False)
+            token = click.prompt('Enter root token', hide_input=True, confirmation_prompt=False)
         assert token is not None
         resp = requests.post("{}/api/v0.1/secrets/auto_seal".format(address.strip("/")),
                              headers={"Inner-Token": token})

cmdb-api/api/lib/cmdb/attribute.py

@@ -415,7 +415,7 @@ def update(self, _id, **kwargs):
             db.session.rollback()
             current_app.logger.error("update attribute error, {0}".format(str(e)))
 
-            return abort(400, ErrFormat.update_attribute_failed.format(("id=".format(_id))))
+            return abort(400, ErrFormat.update_attribute_failed.format(("id={}".format(_id))))
 
         new = attr.to_dict()
         if not new['choice_web_hook'] and new['is_choice']:

cmdb-api/api/lib/cmdb/ci.py

@@ -524,10 +524,14 @@ def update(self, ci_id, _is_admin=False, ticket_id=None, _sync=False, **ci_dict)
         raw_dict = copy.deepcopy(ci_dict)
 
         ci_attr2type_attr = {type_attr.attr_id: type_attr for type_attr, _ in attrs}
+        unique_name = None
         for _, attr in attrs:
             if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT:
                 ci_dict[attr.name] = now
 
+            if attr.id == ci_type.unique_id:
+                unique_name = attr.name
+
         value_manager = AttributeValueManager()
 
         password_dict = dict()
@@ -557,7 +561,8 @@ def update(self, ci_id, _is_admin=False, ticket_id=None, _sync=False, **ci_dict)
 
             ci_dict = {k: v for k, v in ci_dict.items() if k in ci_type_attrs_name}
             key2attr = value_manager.valid_attr_value(ci_dict, ci.type_id, ci.id, ci_type_attrs_name,
-                                                      ci_attr2type_attr=ci_attr2type_attr)
+                                                      ci_attr2type_attr=ci_attr2type_attr,
+                                                      unique_name=unique_name)
 
             if computed_attrs:
                 value_manager.handle_ci_compute_attributes(ci_dict, computed_attrs, ci)

cmdb-api/api/lib/cmdb/ci_type.py

@@ -722,9 +722,6 @@ def delete(cls, type_id, attr_ids=None):
 
                     ci_cache.apply_async(args=(ci.id, None, None), queue=CMDB_QUEUE)
 
-                for item in PreferenceShowAttributes.get_by(type_id=type_id, attr_id=attr_id, to_dict=False):
-                    item.soft_delete(commit=False)
-
                 child_ids = CITypeInheritanceManager.recursive_children(type_id)
                 for _type_id in [type_id] + child_ids:
                     for item in CITypeUniqueConstraint.get_by(type_id=_type_id, to_dict=False):
@@ -740,6 +737,9 @@ def delete(cls, type_id, attr_ids=None):
                     item = CITypeTrigger.get_by(type_id=_type_id, attr_id=attr_id, to_dict=False, first=True)
                     item and item.soft_delete(commit=False)
 
+                    for item in PreferenceShowAttributes.get_by(type_id=_type_id, attr_id=attr_id, to_dict=False):
+                        item.soft_delete(commit=False)
+
                 for item in (CITypeRelation.get_by(parent_id=type_id, to_dict=False) +
                              CITypeRelation.get_by(child_id=type_id, to_dict=False)):
                     if item.parent_id == type_id and attr_id in (item.parent_attr_ids or []):
@@ -862,15 +862,15 @@ def find_path(source_type_id, target_type_ids):
 
         graph = nx.DiGraph()
 
-        def get_children(_id):
+        def get_children(_id, _graph):
             children = CITypeRelation.get_by(parent_id=_id, to_dict=False)
 
             for i in children:
                 if i.child_id != _id:
-                    graph.add_edge(i.parent_id, i.child_id)
-                    get_children(i.child_id)
+                    _graph.add_edge(i.parent_id, i.child_id)
+                    get_children(i.child_id, _graph)
 
-        get_children(source_type_id)
+        get_children(source_type_id, graph)
 
         paths = list(nx.all_simple_paths(graph, source_type_id, target_type_ids))
 
@@ -1145,13 +1145,14 @@ def get_by_type_id(type_id, need_other=False):
             else:
                 group_pos = group2pos[group['name']]
 
-            attr = None
             for i in items:
                 if i.attr_id in id2attr:
                     attr = id2attr[i.attr_id]
                     attr['inherited'] = group['inherited']
                     attr['inherited_from'] = group.get('inherited_from')
                     result[group_pos]['attributes'].append(attr)
+                else:
+                    continue
 
                 if i.attr_id in attr2pos:
                     result[attr2pos[i.attr_id][0]]['attributes'].remove(attr2pos[i.attr_id][1])

cmdb-api/api/lib/cmdb/ipam/history.py

@@ -43,7 +43,7 @@ def add(self, **kwargs):
 
         if kwargs.get('ips'):
             from api.lib.cmdb.ipam.address import IpAddressManager
-            IpAddressManager().assign_ips(kwargs['ips'], None, kwargs.get('cidr'),
+            IpAddressManager().assign_ips(kwargs['ips'], ci_id, kwargs.get('cidr'),
                                           **{IPAddressBuiltinAttributes.IS_USED: 1})
 
             scan_rule = IPAMSubnetScan.get_by(ci_id=ci_id, first=True, to_dict=False)

cmdb-api/api/lib/cmdb/resp_format.py

@@ -16,8 +16,9 @@ class ErrFormat(CommonErrFormat):
     argument_file_not_found = _l("The file doesn't seem to be uploaded")  # 文件似乎并未上传
 
     attribute_not_found = _l("Attribute {} does not exist!")  # 属性 {} 不存在!
+    # 该属性是模型的唯一标识，不能被删除!
     attribute_is_unique_id = _l(
-        "This attribute is the unique identifier of the model and cannot be deleted!")  # 该属性是模型的唯一标识，不能被删除!
+        "This attribute is the unique identifier of the model and cannot be deleted!")  
     attribute_is_ref_by_type = _l(
         "This attribute is referenced by model {} and cannot be deleted!")  # 该属性被模型 {} 引用, 不能删除!
     attribute_value_type_cannot_change = _l(
@@ -129,7 +130,8 @@ class ErrFormat(CommonErrFormat):
     adr_default_ref_once = _l("The default auto-discovery rule is already referenced by model {}!")
     # unique_key方法必须返回非空字符串!
     adr_unique_key_required = _l("The unique_key method must return a non-empty string!")
-    adr_plugin_attributes_list_required = _l("The attributes method must return a list")  # attributes方法必须返回的是list
+    # attributes方法必须返回的是list
+    adr_plugin_attributes_list_required = _l("The attributes method must return a list")  
     # attributes方法返回的list不能为空!
     adr_plugin_attributes_list_no_empty = _l("The list returned by the attributes method cannot be empty!")
     # 只有管理员才可以定义执行机器为: 所有节点!

cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py

@@ -107,3 +107,12 @@
     WHERE c_value_index_datetime.value LIKE "{0}") AS {1}
 GROUP BY  {1}.ci_id
 """
+
+QUERY_CI_BY_NO_ATTR_IN = """
+SELECT *
+FROM 
+    (SELECT c_value_index_texts.ci_id
+    FROM c_value_index_texts
+    WHERE c_value_index_texts.value in ({0})) AS {1}
+GROUP BY {1}.ci_id
+"""

cmdb-api/api/lib/cmdb/search/ci/db/search.py

@@ -6,6 +6,7 @@
 import copy
 import six
 import time
+from flask import abort
 from flask import current_app
 from flask_login import current_user
 from jinja2 import Template
@@ -27,6 +28,7 @@
 from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_ATTR_NAME
 from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_ID
 from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_NO_ATTR
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_NO_ATTR_IN
 from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_TYPE
 from api.lib.cmdb.search.ci.db.query_sql import QUERY_UNION_CI_ATTRIBUTE_IS_NULL
 from api.lib.cmdb.utils import TableMap
@@ -141,7 +143,7 @@ def _type_query_handler(self, v, queries, is_sub=False):
                                     if str(ci_type.id) in self.type_id_list:
                                         self.type_id_list.remove(str(ci_type.id))
                                     type_id_list.remove(str(ci_type.id))
-                                    sub.extend([i for i in queries[1:] if isinstance(i, six.string_types)])
+                                    sub.extend([i for i in queries[1:] if isinstance(i, (six.string_types, list))])
 
                                     sub.insert(0, "_type:{}".format(ci_type.id))
                                     queries.append(dict(operator="|", queries=sub))
@@ -151,7 +153,9 @@ def _type_query_handler(self, v, queries, is_sub=False):
                                 if not self.fl:
                                     self.fl = set(self.type2filter_perms[ci_type.id]['attr_filter'])
                                 else:
-                                    self.fl = set(self.fl) & set(self.type2filter_perms[ci_type.id]['attr_filter'])
+                                    fl = set(self.fl) & set(self.type2filter_perms[ci_type.id]['attr_filter'])
+                                    not fl and abort(400, ErrFormat.ci_filter_perm_attr_no_permission.format(self.fl))
+                                    self.fl = fl
                             else:
                                 self.fl = self.fl or {}
                                 if not self.fl or isinstance(self.fl, dict):
@@ -433,11 +437,14 @@ def __confirm_type_first(self, queries):
                 if not q.startswith("("):
                     raise SearchError(ErrFormat.ci_search_Parentheses_invalid)
 
-                operator, q = self._operator_proc(q)
-                if q.endswith(")"):
-                    result.append(dict(operator=operator, queries=[q[1:-1]]))
+                if ":" not in q:  # multi-line search
+                    result.append(q[1:-1].split(';'))
+                else:
+                    operator, q = self._operator_proc(q)
+                    if q.endswith(")"):
+                        result.append(dict(operator=operator, queries=[q[1:-1]]))
 
-                sub = dict(operator=operator, queries=[q[1:]])
+                    sub = dict(operator=operator, queries=[q[1:]])
             elif q.endswith(")") and sub:
                 sub['queries'].append(q[:-1])
                 result.append(copy.deepcopy(sub))
@@ -525,22 +532,31 @@ def __query_build_by_field(self, queries, is_first=True, only_type_query_special
         query_sql = ""
 
         for q in queries:
+            # current_app.logger.debug(q)
             _query_sql = ""
             if isinstance(q, dict):
-                alias, _query_sql, operator = self.__query_build_by_field(q['queries'], True, True, alias, is_sub=True)
-                # current_app.logger.info(_query_sql)
-                # current_app.logger.info((operator, is_first, alias))
-                operator = q['operator']
+                if len(q['queries']) == 1 and ";" in q['queries'][0]:
+                    values = q['queries'][0].split(";")
+                    in_values = ",".join("'{0}'".format(v) for v in values)
+                    _query_sql = QUERY_CI_BY_NO_ATTR_IN.format(in_values, alias)
+                    operator = q['operator']
+                else:
+                    alias, _query_sql, operator = self.__query_build_by_field(q['queries'], True, True, alias,
+                                                                              is_sub=True)
+                    operator = q['operator']
 
             elif ":" in q and not q.startswith("*"):
                 alias, _query_sql, operator = self.__query_by_attr(q, queries, alias, is_sub)
             elif q == "*":
                 continue
             elif q:
-                q = q.replace("'", "\\'")
-                q = q.replace('"', '\\"')
-                q = q.replace("*", "%").replace('\\n', '%')
-                _query_sql = QUERY_CI_BY_NO_ATTR.format(q, alias)
+                if not isinstance(q, list):
+                    q = q.replace("'", "\\'")
+                    q = q.replace('"', '\\"')
+                    q = q.replace("*", "%").replace('\\n', '%')
+                    _query_sql = QUERY_CI_BY_NO_ATTR.format(q, alias)
+                else:
+                    _query_sql = QUERY_CI_BY_NO_ATTR_IN.format(",".join("'{0}'".format(v) for v in q), alias)
 
             if is_first and _query_sql and not self.only_type_query:
                 query_sql = "SELECT * FROM ({0}) AS {1}".format(_query_sql, alias)

cmdb-api/api/lib/cmdb/utils.py

@@ -55,7 +55,6 @@ def str2datetime(x):
     return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M")
 
 
-
 class ValueTypeMap(object):
     deserialize = {
         ValueTypeEnum.INT: string2int,

cmdb-api/api/lib/cmdb/value.py

@@ -136,7 +136,7 @@ def check_re(expr, alias, value):
         if not re.compile(expr).match(str(value)):
             return abort(400, ErrFormat.attribute_value_invalid2.format(alias, value))
 
-    def _validate(self, attr, value, value_table, ci=None, type_id=None, ci_id=None, type_attr=None):
+    def _validate(self, attr, value, value_table, ci=None, type_id=None, ci_id=None, type_attr=None, unique_name=None):
         if not attr.is_reference:
             ci = ci or {}
             v = self._deserialize_value(attr.alias, attr.value_type, value)
@@ -146,7 +146,7 @@ def _validate(self, attr, value, value_table, ci=None, type_id=None, ci_id=None,
         else:
             v = value or None
 
-        attr.is_unique and self._check_is_unique(
+        (attr.is_unique or attr.name == unique_name) and self._check_is_unique(
             value_table, attr, ci and ci.id or ci_id, ci and ci.type_id or type_id, v)
         self._check_is_required(ci and ci.type_id or type_id, attr, v, type_attr=type_attr)
         if attr.is_reference:
@@ -237,7 +237,10 @@ def handle_ci_compute_attributes(self, ci_dict, computed_attrs, ci):
             if computed_value is not None:
                 ci_dict[attr['name']] = computed_value
 
-    def valid_attr_value(self, ci_dict, type_id, ci_id, name2attr, alias2attr=None, ci_attr2type_attr=None):
+    def valid_attr_value(self, ci_dict, type_id, ci_id, name2attr,
+                         alias2attr=None,
+                         ci_attr2type_attr=None,
+                         unique_name=None):
         key2attr = dict()
         alias2attr = alias2attr or {}
         ci_attr2type_attr = ci_attr2type_attr or {}
@@ -268,7 +271,8 @@ def valid_attr_value(self, ci_dict, type_id, ci_id, name2attr, alias2attr=None,
 
                 else:
                     value = self._validate(attr, value, value_table, ci=None, type_id=type_id, ci_id=ci_id,
-                                           type_attr=ci_attr2type_attr.get(attr.id))
+                                           type_attr=ci_attr2type_attr.get(attr.id),
+                                           unique_name=unique_name)
                     ci_dict[key] = value
             except BadRequest as e:
                 raise