Merge branch 'master' into dev

CHANGES.rst

@@ -1,6 +1,18 @@
-4.2.2.dev0 (Next Release)
+4.3.0.dev0 (Next Release)
 -------------------------
 
+- The web interface will now return a 404 Not Found response if a log file
+  is missing.  Previously, it would return 410 Gone.  It was changed because
+  410 is intended to mean that the condition is likely to be permanent.  A
+  log file missing is usually temporary, e.g. a process that was never started
+  will not have a log file but will have one as soon as it is started.
+
+4.2.2 (2021-02-26)
+------------------
+
+- Fixed a bug where ``supervisord`` could crash if a subprocess exited
+  immediately before trying to kill it.
+
 - Fixed a bug where the ``stdout_syslog`` and ``stderr_syslog`` options
   of a ``[program:x]`` section could not be used unless file logging for
   the same program had also been configured.  The file and syslog options
@@ -11,9 +23,16 @@
   ``syslog`` was supplied, as is supported by all other log filename
   options.  Patch by Franck Cuny.
 
+- Fixed a bug where environment variables defined in ``environment=``
+  in the ``[supervisord]`` section or a ``[program:x]`` section could
+  not be used in ``%(ENV_x)s`` expansions.  Patch by MythRen.
+
 - The  ``supervisorctl signal`` command now allows a signal to be sent
   when a process is in the ``STOPPING`` state.  Patch by Mike Gould.
 
+- ``supervisorctl`` and ``supervisord`` now print help when given ``-?``
+  in addition to the existing ``-h``/``--help``.
+
 4.2.1 (2020-08-20)
 ------------------
 

docs/conf.py

@@ -22,14 +22,6 @@
 
 parent = os.path.dirname(os.path.dirname(__file__))
 sys.path.append(os.path.abspath(parent))
-wd = os.getcwd()
-os.chdir(parent)
-os.system('%s setup.py test -q' % sys.executable)
-os.chdir(wd)
-
-for item in os.listdir(parent):
-    if item.endswith('.egg'):
-        sys.path.append(os.path.join(parent, item))
 
 version_txt = os.path.join(parent, 'supervisor/version.txt')
 supervisor_version = open(version_txt).read().strip()

docs/configuration.rst

@@ -445,8 +445,8 @@ follows.
 ``environment``
 
   A list of key/value pairs in the form ``KEY="val",KEY2="val2"`` that
-  will be placed in the :program:`supervisord` process' environment
-  (and as a result in all of its child process' environments).  This
+  will be placed in the environment of all child processes.  This does
+  not change the environment of :program:`supervisord` itself.  This
   option can include the value ``%(here)s``, which expands to the
   directory in which the supervisord configuration file was found.
   Values containing non-alphanumeric characters should be quoted
@@ -752,8 +752,15 @@ where specified.
 
   The number of serial failure attempts that :program:`supervisord`
   will allow when attempting to start the program before giving up and
-  putting the process into an ``FATAL`` state.  See
-  :ref:`process_states` for explanation of the ``FATAL`` state.
+  putting the process into an ``FATAL`` state.
+
+  .. note::
+
+      After each failed restart, process will be put in ``BACKOFF`` state
+      and each retry attempt will take increasingly more time.
+
+      See :ref:`process_states` for explanation of the ``FATAL`` and
+      ``BACKOFF`` states.
 
   *Default*: 3
 
@@ -1525,8 +1532,8 @@ Adding ``rpcinterface:x`` settings in the configuration file is only
 useful for people who wish to extend supervisor with additional custom
 behavior.
 
-In the sample config file, there is a section which is named
-``[rpcinterface:supervisor]``.  By default it looks like the
+In the sample config file (see :ref:`create_config`), there is a section
+which is named ``[rpcinterface:supervisor]``.  By default it looks like the
 following.
 
 .. code-block:: ini

docs/installing.rst

@@ -101,6 +101,8 @@ the system boots.
     `Ubuntu Bug #1594740 <https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740>`_
     for more information.
 
+.. _create_config:
+
 Creating a Configuration File
 -----------------------------
 

docs/plugins.rst

@@ -184,3 +184,5 @@ with third party applications:
 `supervisor-alert <https://github.com/rahiel/supervisor-alert>`_
     Send event notifications over `Telegram <https://telegram.org>`_ or to an
     arbitrary command.
+`supervisor-discord <https://github.com/chaos-a/supervisor-discord>`_
+    Send event notifications to `Discord <https://discord.com>`_ via webhooks.

docs/subprocess.rst

@@ -218,7 +218,8 @@ interface elements in clients.
 ``BACKOFF`` (30)
 
   The process entered the ``STARTING`` state but subsequently exited
-  too quickly to move to the ``RUNNING`` state.
+  too quickly (before the time defined in ``startsecs``) to move to
+  the ``RUNNING`` state.
 
 ``STOPPING`` (40)
 
@@ -254,8 +255,15 @@ automatically restarted by :program:`supervisord`.  It will switch
 between ``STARTING`` and ``BACKOFF`` states until it becomes evident
 that it cannot be started because the number of ``startretries`` has
 exceeded the maximum, at which point it will transition to the
-``FATAL`` state.  Each start retry will take progressively
-more time.
+``FATAL`` state.
+
+.. note::
+    Retries will take increasingly more time depending on the number of
+    subsequent attempts made, adding one second each time.
+
+    So if you set ``startretries=3``, :program:`supervisord` will wait one,
+    two and then three seconds between each restart attempt, for a total of
+    5 seconds.
 
 When a process is in the ``EXITED`` state, it will
 automatically restart:

supervisor/http.py

@@ -737,10 +737,11 @@ def handle_request(self, request):
         logfile = getattr(process.config, '%s_logfile' % channel, None)
 
         if logfile is None or not os.path.exists(logfile):
-            # XXX problematic: processes that don't start won't have a log
-            # file and we probably don't want to go into fatal state if we try
-            # to read the log of a process that did not start.
-            request.error(410) # gone
+            # we return 404 because no logfile is a temporary condition.
+            # if the process has never been started, no logfile will exist
+            # on disk.  a logfile of None is also a temporay condition,
+            # since the config file can be reloaded.
+            request.error(404) # not found
             return
 
         mtime = os.stat(logfile)[stat.ST_MTIME]
@@ -774,7 +775,10 @@ def handle_request(self, request):
         logfile = self.supervisord.options.logfile
 
         if logfile is None or not os.path.exists(logfile):
-            request.error(410) # gone
+            # we return 404 because no logfile is a temporary condition.
+            # even if a log file of None is configured, the config file
+            # may be reloaded, and the new config may have a logfile.
+            request.error(404) # not found
             return
 
         mtime = os.stat(logfile)[stat.ST_MTIME]

supervisor/options.py

@@ -102,6 +102,7 @@ def __init__(self, require_configfile=True):
         self.attr_priorities = {}
         self.require_configfile = require_configfile
         self.add(None, None, "h", "help", self.help)
+        self.add(None, None, "?", None, self.help)
         self.add("configfile", None, "c:", "configuration=")
 
         here = os.path.dirname(os.path.dirname(sys.argv[0]))
@@ -405,7 +406,6 @@ class ServerOptions(Options):
     passwdfile = None
     nodaemon = None
     silent = None
-    environment = None
     httpservers = ()
     unlink_pidfile = False
     unlink_socketfiles = False
@@ -656,6 +656,11 @@ def get(opt, default, **kwargs):
         environ_str = get('environment', '')
         environ_str = expand(environ_str, expansions, 'environment')
         section.environment = dict_of_key_value_pairs(environ_str)
+
+        # extend expansions for global from [supervisord] environment definition
+        for k, v in section.environment.items():
+            self.environ_expansions['ENV_%s' % k ] = v
+
         # Process rpcinterface plugins before groups to allow custom events to
         # be registered.
         section.rpcinterface_factories = self.get_plugins(
@@ -963,6 +968,10 @@ def get(section, opt, *args, **kwargs):
             environment = dict_of_key_value_pairs(
                 expand(environment_str, expansions, 'environment'))
 
+            # extend expansions for process from [program:x] environment definition
+            for k, v in environment.items():
+                expansions['ENV_%s' % k] = v
+
             directory = get(section, 'directory', None)
 
             logfiles = {}
@@ -1589,9 +1598,6 @@ def readfd(self, fd):
             data = b''
         return data
 
-    def process_environment(self):
-        os.environ.update(self.environment or {})
-
     def chdir(self, dir):
         os.chdir(dir)
 

supervisor/pidproxy.py

@@ -1,7 +1,13 @@
-#!/usr/bin/env python
+#!/usr/bin/env python -u
 
-""" An executable which proxies for a subprocess; upon a signal, it sends that
-signal to the process identified by a pidfile. """
+"""pidproxy -- run command and proxy signals to it via its pidfile.
+
+This executable runs a command and then monitors a pidfile.  When this
+executable receives a signal, it sends the same signal to the pid
+in the pidfile.
+
+Usage: %s <pidfile name> <command> [<cmdarg1> ...]
+"""
 
 import os
 import sys
@@ -10,18 +16,19 @@
 
 class PidProxy:
     pid = None
+
     def __init__(self, args):
-        self.setsignals()
         try:
             self.pidfile, cmdargs = args[1], args[2:]
-            self.command = os.path.abspath(cmdargs[0])
+            self.abscmd = os.path.abspath(cmdargs[0])
             self.cmdargs = cmdargs
         except (ValueError, IndexError):
             self.usage()
             sys.exit(1)
 
     def go(self):
-        self.pid = os.spawnv(os.P_NOWAIT, self.command, self.cmdargs)
+        self.setsignals()
+        self.pid = os.spawnv(os.P_NOWAIT, self.abscmd, self.cmdargs)
         while 1:
             time.sleep(5)
             try:
@@ -32,7 +39,7 @@ def go(self):
                 break
 
     def usage(self):
-        print("pidproxy.py <pidfile name> <command> [<cmdarg1> ...]")
+        print(__doc__ % sys.argv[0])
 
     def setsignals(self):
         signal.signal(signal.SIGTERM, self.passtochild)
@@ -64,6 +71,3 @@ def main():
 
 if __name__ == '__main__':
     main()
-
-
-

supervisor/process.py

@@ -1,10 +1,10 @@
-import os
-import time
 import errno
+import functools
+import os
+import signal
 import shlex
+import time
 import traceback
-import signal
-from functools import total_ordering
 
 from supervisor.compat import maxint
 from supervisor.compat import as_bytes
@@ -30,7 +30,7 @@
 
 from supervisor.socket_manager import SocketManager
 
-@total_ordering
+@functools.total_ordering
 class Subprocess(object):
 
     """A class to manage a subprocess."""
@@ -403,7 +403,8 @@ def give_up(self):
         self.change_state(ProcessStates.FATAL)
 
     def kill(self, sig):
-        """Send a signal to the subprocess.  This may or may not kill it.
+        """Send a signal to the subprocess with the intention to kill
+        it (to make it exit).  This may or may not actually kill it.
 
         Return None if the signal was sent, or an error message string
         if an error occurred or if the subprocess is not running.
@@ -463,14 +464,23 @@ def kill(self, sig):
             pid = -self.pid
 
         try:
-            options.kill(pid, sig)
+            try:
+                options.kill(pid, sig)
+            except OSError as exc:
+                if exc.errno == errno.ESRCH:
+                    msg = ("unable to signal %s (pid %s), it probably just exited "
+                           "on its own: %s" % (processname, self.pid, str(exc)))
+                    options.logger.debug(msg)
+                    # we could change the state here but we intentionally do
+                    # not.  we will do it during normal SIGCHLD processing.
+                    return None
+                raise
         except:
             tb = traceback.format_exc()
             msg = 'unknown problem killing %s (%s):%s' % (processname,
                                                           self.pid, tb)
             options.logger.critical(msg)
             self.change_state(ProcessStates.UNKNOWN)
-            self.pid = 0
             self.killing = False
             self.delay = 0
             return msg
@@ -502,14 +512,24 @@ def signal(self, sig):
                             ProcessStates.STOPPING)
 
         try:
-            options.kill(self.pid, sig)
+            try:
+                options.kill(self.pid, sig)
+            except OSError as exc:
+                if exc.errno == errno.ESRCH:
+                    msg = ("unable to signal %s (pid %s), it probably just now exited "
+                           "on its own: %s" % (processname, self.pid,
+                           str(exc)))
+                    options.logger.debug(msg)
+                    # we could change the state here but we intentionally do
+                    # not.  we will do it during normal SIGCHLD processing.
+                    return None
+                raise
         except:
             tb = traceback.format_exc()
             msg = 'unknown problem sending sig %s (%s):%s' % (
                                 processname, self.pid, tb)
             options.logger.critical(msg)
             self.change_state(ProcessStates.UNKNOWN)
-            self.pid = 0
             return msg
 
         return None
@@ -753,7 +773,7 @@ def _prepare_child_fds(self):
         for i in range(3, options.minfds):
             options.close_fd(i)
 
-@total_ordering
+@functools.total_ordering
 class ProcessGroupBase(object):
     def __init__(self, config):
         self.config = config

supervisor/rpcinterface.py

@@ -299,6 +299,10 @@ def startProcess(self, name, wait=True):
         if process.get_state() in RUNNING_STATES:
             raise RPCError(Faults.ALREADY_STARTED, name)
 
+        if process.get_state() == ProcessStates.UNKNOWN:
+            raise RPCError(Faults.FAILED,
+                           "%s is in an unknown process state" % name)
+
         process.spawn()
 
         # We call reap() in order to more quickly obtain the side effects of

supervisor/supervisord.py

@@ -84,7 +84,6 @@ def run(self):
         try:
             for config in self.options.process_group_configs:
                 self.add_process_group(config)
-            self.options.process_environment()
             self.options.openhttpservers(self)
             self.options.setsignals()
             if (not self.options.nodaemon) and self.options.first:
@@ -96,9 +95,8 @@ def run(self):
         finally:
             self.options.cleanup()
 
-    def diff_to_active(self, new=None):
-        if not new:
-            new = self.options.process_group_configs
+    def diff_to_active(self):
+        new = self.options.process_group_configs
         cur = [group.config for group in self.process_groups.values()]
 
         curdict = dict(zip([cfg.name for cfg in cur], cur))