✨ add webhook and dingding notice

dkops.yaml

@@ -1,6 +1,6 @@
 common:
   restart_timeout: 600
-  interval: 5
+  interval: 10 
   proc_name: dkops
   log_level: info
   log_file: ./log/dkops.log
@@ -10,3 +10,8 @@ default_limit:
 container_limit:
   docker_test: 
     memory: 0
+
+notice:
+  # method: webhook or dingding
+  method: webhook
+  url: http://localhost:8888/webhook

tools/container_helper.py

@@ -1,14 +1,20 @@
 #!/usr/bin/env python
-from utils.helper import calc_mem_precent
-from utils.docker_helper import docker_client
 import logging
 import sys
+import traceback
+
+from utils.docker_helper import docker_client
+from utils.helper import calc_mem_precent
+from utils.message_send_helper import msg_sender
 
 
 logger = logging.getLogger("main")
 
 
-def container_handler(config, container_info):
+def container_handler(config, container_size_info):
+    container_info = docker_client.stats(container_size_info["name"], stream=False)
+    container_info["SizeRw"] = container_size_info["SizeRw"]
+    container_info["SizeRootFs"] = container_size_info["SizeRootFs"]
     container_name = container_info.get("name", "").strip("/")
     if container_name in config.get("container_limit"):
         limit_config = config["container_limit"][container_name]
@@ -22,12 +28,23 @@ def container_handler(config, container_info):
             sys.exit(0)
         logger.info("container {} memory limit {}%".format(container_name, mem_limit))
         if calc_mem_precent(container_info) >= mem_limit:
-            logger.warn(
-                "container {} memory over limit. restart it".format(container_name)
-            )
-            docker_client.restart(
-                container_info["id"], config["common"]["restart_timeout"]
+            first_msg = "container {} memory over limit. restart it".format(
+                container_name
             )
+            logger.warn(first_msg)
+            msg_sender(config, first_msg, container_info)
+            try:
+                docker_client.restart(
+                    container_info["id"],
+                    config.get("common", {}).get("restart_timeout", 600),
+                )
+            except Exception:
+                logging.error(traceback.format_exc())
+                end_msg = "container {} restart failed!".format(container_name)
+            else:
+                end_msg = "container {} restart successfully!".format(container_name)
+            logger.warn(end_msg)
+            msg_sender(config, end_msg, container_info)
         else:
             logger.info("container {} memory is normal.".format(container_name))
     else:

tools/job.py

@@ -1,13 +1,16 @@
 #!/usr/bin/env python
 import logging
+import threading
+import random
+
 from tools.container_helper import container_handler
 from utils.docker_helper import docker_client
 
 logger = logging.getLogger("main")
 
 
 def docker_monitor(config):
-    logger.info(config)
+    logger.info("系统配置: {}".format(config))
     logger.info("docker version: {} ".format(docker_client.version().get("Version")))
     containers = docker_client.containers(size=True)
     if not containers:
@@ -17,7 +20,17 @@ def docker_monitor(config):
             container_name = item.get("Names")[0]
             SizeRw = item.get("SizeRw", 0)
             SizeRootFs = item.get("SizeRootFs", 0)
-            docker_stats = docker_client.stats(container_name, stream=False)
-            docker_stats["SizeRw"] = SizeRw
-            docker_stats["SizeRootFs"] = SizeRootFs
-            container_handler(config, docker_stats)
+            container_size_info = {
+                "SizeRootFs": SizeRootFs,
+                "SizeRw": SizeRw,
+                "name": container_name,
+            }
+            docker_thread = threading.Thread(
+                target=container_handler,
+                daemon=True,
+                args=(config, container_size_info),
+                name="{}_container_thread_{}".format(
+                    container_name.strip("/"), random.randint(0, 10000)
+                ),
+            )
+            docker_thread.start()

utils/message_send_helper.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+import logging
+
+from utils import notice
+
+logger = logging.getLogger("main")
+
+
+def msg_sender(config, msg, docker_info):
+    # 获取配置
+    notice_config = config.get("notice")
+    if not notice_config:
+        logger.warn("未发现notice配置")
+        return
+    notice_method = notice_config.get("method")
+    if not notice_method:
+        logger.warn("未发现notice配置")
+        return
+
+    if not hasattr(notice, notice_method):
+        logger.warn("notice方式不存在. 请检查配置")
+        return
+    timeout = notice_config.get("timeout", 10)
+    msg_func = getattr(notice, notice_method)
+    kwargs = {}
+    if notice_method == "dingding":
+        kwargs["timeout"] = timeout
+        kwargs["token"] = notice_config["token"]
+        kwargs["msg"] = msg
+    elif notice_method == "webhook":
+        docker_info["msg"] = msg
+
+        kwargs["timeout"] = timeout
+        kwargs["url"] = notice_config["url"]
+        kwargs["hookinfo"] = docker_info
+    else:
+        logger.warn("notice方式不存在. 请检查配置")
+        return
+    msg_func(**kwargs)

utils/notice.py

@@ -22,5 +22,13 @@ def dingding(token, msg, timeout=10):
         logger.error(traceback.format_exc())
 
 
-def webhooks(hookinfo):
-    pass
+def webhook(url, hookinfo, timeout=10):
+    try:
+        webhooks_res = requests.post(url, json=hookinfo, timeout=timeout)
+        if webhooks_res.status_code == 200:
+            logger.info("webhooks发送成功")
+        else:
+            logger.error("webhook消息发送失败")
+    except Exception:
+        logger.error("webhook消息发送失败")
+        logger.error(traceback.format_exc())