Add support for supervisord (hyperledger#588)

* Adding support for supervisord which requires setting the INDY_CONTROL environment variable to supervisorctl Signed-off-by: Keith Smith <[email protected]> * Fixes per review from andkononykhin Signed-off-by: Keith Smith <[email protected]> * More changes per Andrey's review Signed-off-by: Keith Smith <[email protected]>
lucianoacsilva · Apr 13, 2018 · 31c0328 · 31c0328
1 parent 0400e35
commit 31c0328
Show file tree

Hide file tree

Showing 5 changed files with 163 additions and 13 deletions.
diff --git a/build-scripts/ubuntu-1604/postinst_node b/build-scripts/ubuntu-1604/postinst_node
@@ -114,7 +114,7 @@ After=network.target
 [Service]
 Type=simple
 EnvironmentFile=$GENERAL_CONFIG_DIR/node_control.conf
-ExecStart=/usr/bin/env python3 -O /usr/local/bin/start_node_control_tool \$TEST_MODE --hold-ext \$HOLD_EXT
+ExecStart=/usr/bin/env python3 -O /usr/local/bin/start_node_control_tool \${TEST_MODE} --hold-ext "\${HOLD_EXT}"
 Restart=on-failure
 RestartSec=10
 StartLimitBurst=10
@@ -125,14 +125,57 @@ TimeoutSec=300
 WantedBy=multi-user.target
 EOF
 
+# add supervisord script
+cat <<EOF > /etc/supervisor/indy-node.conf
+[supervisord]
+nodaemon=true
+logfile=/var/log/indy/supervisord.log
+logfile_maxbytes=10MB
+loglevel=critical
+
+[supervisorctl]
+serverurl=http://127.0.0.1:9001
+
+[inet_http_server]
+port = 127.0.0.1:9001
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[program:indy-node]
+command=sh -ac '. $GENERAL_CONFIG_DIR/indy.env;/usr/bin/env python3 -O /usr/local/bin/start_indy_node \${NODE_NAME} \${NODE_PORT} \${NODE_CLIENT_PORT}'
+user=indy
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=true
+startsecs=15
+startretries=6
+stopasgroup=true
+killasgroup=true
+
+[program:indy-node-control]
+command=sh -ac '. $GENERAL_CONFIG_DIR/node_control.conf;/usr/bin/env python3 -O /usr/local/bin/start_node_control_tool \${TEST_MODE} --hold-ext "\${HOLD_EXT}"'
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=true
+startsecs=15
+startretries=6
+stopasgroup=true
+killasgroup=true
+EOF
+
 HOLD_EXT_ADDED=$(grep HOLD_EXT /etc/indy/node_control.conf)
 if [ ! -f $GENERAL_CONFIG_DIR/node_control.conf ] || [ -z "${HOLD_EXT_ADDED}" ]; then
     cat <<EOF > $GENERAL_CONFIG_DIR/node_control.conf
 # Uncomment this to run agent in test mode:
 #TEST_MODE=--test
 
 TEST_MODE=
-HOLD_EXT=\"\"
+HOLD_EXT=
 EOF
 fi
 

diff --git a/scripts/restart_indy_node_ubuntu1604.sh b/scripts/restart_indy_node_ubuntu1604.sh
@@ -1,11 +1,32 @@
 #!/bin/bash -x
 
-# Upgrade may change service files
-systemctl daemon-reload
-systemctl reset-failed
+if [ "$INDY_CONTROL" = "" ]; then
 
-echo "Starting indy-node"
-systemctl start indy-node
+  # Upgrade may change service files
+  systemctl daemon-reload
+  systemctl reset-failed
 
-echo "Restarting agent"
-systemctl restart indy-node-control
+  echo "Starting indy-node"
+  systemctl start indy-node
+
+  echo "Restarting agent"
+  systemctl restart indy-node-control
+
+elif [ "$INDY_CONTROL" = "supervisorctl" ]; then
+
+  # Upgrade may change service files
+  supervisorctl reread
+  supervisorctl update
+
+  echo "Starting indy-node"
+  supervisorctl start indy-node
+
+  echo "Restarting agent"
+  supervisorctl restart indy-node-control
+
+else
+
+  echo "Invalid setting for 'INDY_CONTROL' environment variable: $INDY_CONTROL"
+  exit 1
+
+fi
diff --git a/scripts/upgrade_indy_node_ubuntu1604.sh b/scripts/upgrade_indy_node_ubuntu1604.sh
@@ -13,8 +13,23 @@ if [ $ret -ne 0 ] ; then
   exit 1
 fi
 
-echo "Stop indy-node"
-systemctl stop indy-node
+if [ "$INDY_CONTROL" = "" ]; then
+
+  echo "Stop indy-node"
+  systemctl stop indy-node
+
+elif [ "$INDY_CONTROL" = "supervisorctl" ]; then
+
+  echo "Stop indy-node"
+  supervisorctl stop indy-node
+
+else
+
+  echo "Invalid setting for 'INDY_CONTROL' environment variable: $INDY_CONTROL"
+  exit 1
+
+fi
+
 
 echo "Run indy upgrade to $deps"
 apt-get -y --allow-downgrades --allow-change-held-packages --reinstall install $deps

diff --git a/scripts/upgrade_indy_node_ubuntu1604_test.sh b/scripts/upgrade_indy_node_ubuntu1604_test.sh
@@ -13,8 +13,23 @@ if [ $ret -ne 0 ] ; then
   exit 1
 fi
 
-echo "Stop indy-node"
-systemctl stop indy-node
+if [ "$INDY_CONTROL" = "" ]; then
+
+  echo "Stop indy-node"
+  systemctl stop indy-node
+
+elif [ "$INDY_CONTROL" = "supervisorctl" ]; then
+
+  echo "Stop indy-node"
+  supervisorctl stop indy-node
+
+else
+
+  echo "Invalid setting for 'INDY_CONTROL' environment variable: $INDY_CONTROL"
+  exit 1
+
+fi
+
 
 echo "Run indy upgrade to $deps"
 apt-get -y --allow-downgrades --allow-change-held-packages --reinstall install $deps

diff --git a/scripts/validator-info b/scripts/validator-info
@@ -351,6 +351,16 @@ class ValidatorStats(BaseStats):
 
     @staticmethod
     def get_process_state():
+        ctl = os.getenv('INDY_CONTROL', 'systemctl')
+        if ctl == "systemctl":
+           return ValidatorStats.get_process_state_via_systemctl()
+        elif ctl == "supervisorctl":
+           return ValidatorStats.get_process_state_via_supervisorctl()
+        else:
+           return "Invalid value for INDY_CONTROL environment variable: '%s'" % ctl
+
+    @staticmethod
+    def get_process_state_via_systemctl():
         ret = subprocess.check_output(
             'systemctl is-failed indy-node; exit 0',
             stderr=subprocess.STDOUT, shell=True
@@ -367,8 +377,36 @@ class ValidatorStats(BaseStats):
             )
             return None
 
+    @staticmethod
+    def get_process_state_via_supervisorctl():
+        ret = subprocess.check_output(
+            "supervisorctl status indy-node | awk '{print $2}'; exit 0",
+            stderr=subprocess.STDOUT, shell=True
+        )
+        ret = ret.decode().strip()
+        if ret == 'STOPPED':
+            return 'stopped'
+        elif ret == 'RUNNING':
+            return 'running'
+        else:
+            logger.info(
+                "Non-expected output for indy-node "
+                "status: {}".format(ret)
+            )
+            return None
+
     @staticmethod
     def get_enabled_state():
+        ctl = os.getenv('INDY_CONTROL', 'systemctl')
+        if ctl == "systemctl":
+           return ValidatorStats.get_enabled_state_via_systemctl()
+        elif ctl == "supervisorctl":
+           return ValidatorStats.get_enabled_state_via_supervisorctl()
+        else:
+           return "Invalid value for INDY_CONTROL environment variable: '%s'" % ctl
+
+    @staticmethod
+    def get_enabled_state_via_systemctl():
         ret = subprocess.check_output(
             'systemctl is-enabled indy-node; exit 0',
             stderr=subprocess.STDOUT, shell=True
@@ -385,6 +423,24 @@ class ValidatorStats(BaseStats):
             )
             return None
 
+    @staticmethod
+    def get_enabled_state_via_supervisorctl():
+        ret = subprocess.check_output(
+            "supervisorctl status indy-node | awk '{print $2}'; exit 0",
+            stderr=subprocess.STDOUT, shell=True
+        )
+        ret = ret.decode().strip()
+        if ret in ('RUNNING', 'BACKOFF', 'STARTING'):
+            return True
+        elif ret == 'STOPPED':
+            return False
+        else:
+            logger.info(
+                "Non-expected output for indy-node "
+                "is-enabled state: {}".format(ret)
+            )
+            return None
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         # TODO move that to classes too