Fix edge case caused by network issues

If extended network issues arise and multiple packets get lost, all servers in the pool could end up losing references to each other and never regaint hem. This now keeps the peers for an hour after disconnecting and keeps pinging them until they come back.

lib/democracy.js

@@ -1,6 +1,6 @@
 /**
  * democracy.js
- * Copyright (c) 2016 - 2018, GoldFire Studios, Inc.
+ * Copyright (c) 2016 - 2019, GoldFire Studios, Inc.
  * http://goldfirestudios.com
  */
 
@@ -306,6 +306,12 @@ class Democracy extends EventEmitter {
       return this;
     }
 
+    // No longer mark this node as removed.
+    if (this._nodes[data.id] && this._nodes[data.id].disconnected) {
+      clearTimeout(this._nodes[data.id].disconnected);
+      delete this._nodes[data.id].disconnected;
+    }
+
     // Process the different available events.
     if (data.event === 'hello') {
       // Create a new node if we don't already know about this one.
@@ -385,13 +391,17 @@ class Democracy extends EventEmitter {
     if (node.voters.length >= numVoters) {
       this.emit('removed', node);
 
-      // Remove from the nodes/peers.
-      const source = node.source.split(':');
-      const index = this.options.peers.findIndex(p => p[0] === source[0] && p[1] === source[1]);
-      if (index >= 0) {
-        this.options.peers.splice(index, 1);
-      }
-      this._nodes[candidate] = null;
+      // Mark the node as removed (to be removed later).
+      this._nodes[candidate].state = 'removed';
+      this._nodes[candidate].disconnected = setTimeout(() => {
+        // Remove from the nodes/peers.
+        const source = node.source.split(':');
+        const index = this.options.peers.findIndex(p => p[0] === source[0] && p[1] === source[1]);
+        if (index >= 0) {
+          this.options.peers.splice(index, 1);
+        }
+        this._nodes[candidate] = null;
+      }, 3600000);
     }
 
     if (state === 'leader') {
@@ -413,7 +423,7 @@ class Democracy extends EventEmitter {
     // Elect a new leader based on highest weight.
     // Each server should always elect the same leader.
     Object.keys(nodes).forEach((id) => {
-      if (nodes[id] && nodes[id].weight > highestWeight) {
+      if (nodes[id] && nodes[id].weight > highestWeight && nodes[id].state !== 'removed') {
         highestWeight = nodes[id].weight;
         newLeader = id;
       }
@@ -425,12 +435,14 @@ class Democracy extends EventEmitter {
     }
 
     // Elect our new benevolent dictator for life...of process.
-    if (newLeader === this._id || !newLeader) {
-      this._state = 'leader';
-      nodes[newLeader].state = 'leader';
-      this.emit('elected', nodes[newLeader]);
+    if (newLeader === this._id) {
+      if (this._state !== 'leader') {
+        this._state = 'leader';
+        nodes[newLeader].state = 'leader';
+        this.emit('elected', nodes[newLeader]);
+      }
       this.send('leader');
-    } else {
+    } else if (newLeader) {
       this._nodes[newLeader].state = 'leader';
     }
 
@@ -462,8 +474,23 @@ class Democracy extends EventEmitter {
    * @return {Object} All nodes.
    */
   nodes() {
+    const nodes = {};
+
+    // Copy the nodes data to return.
+    Object.keys(this._nodes).forEach((id) => {
+      const node = this._nodes[id];
+
+      nodes[node.id] = {
+        id: node.id,
+        weight: node.weight,
+        state: node.state,
+        last: node.last,
+        voters: node.voters,
+        channels: node.channels,
+      };
+    });
+
     // Add this server into the nodes list.
-    const nodes = JSON.parse(JSON.stringify(this._nodes));
     nodes[this._id] = {
       id: this._id,
       weight: this._weight,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "democracy",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "Node.js unicast discovery, master-slave elections and pub/sub.",
   "homepage": "https://github.com/goldfire/democracy.js",
   "keywords": [