net: notice a peer that has gone quiet, and stay audible to peers that check

src/main.cpp

@@ -2190,6 +2190,18 @@ bool ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv)
     }
 
 
+    else if (strCommand == "ping")
+    {
+        // Nothing to do and nothing to answer. Arriving at all is the entire
+        // content of the message: the receive path has already stamped
+        // nLastRecv, which is what the sender wanted us to notice.
+        //
+        // Deliberately no reply. A node old enough not to know "ping" ignores
+        // it as an unknown command and would never answer one, so a scheme
+        // that required an answer would read every such peer as dead.
+    }
+
+
     else
     {
         // Ignore unknown commands for extensibility
@@ -2221,6 +2233,17 @@ bool SendMessages(CNode* pto)
             return true;
 
 
+        //
+        // Message: keep-alive ping
+        //
+        // Only when we have nothing else queued -- any real message serves the
+        // same purpose. A peer with nothing to relay is otherwise silent, and
+        // silence is exactly what the inactivity check disconnects on.
+        //
+        if (pto->nLastSend && GetTime() - pto->nLastSend > BTF_PING_INTERVAL_SECS && pto->vSend.empty())
+            pto->PushMessage("ping");
+
+
         //
         // Message: inventory
         //

src/net.cpp

@@ -1092,7 +1092,11 @@ void ThreadSocketHandler2(void* parg)
                     vRecv.resize(nPos + nBufSize);
                     int nBytes = recv(hSocket, &vRecv[nPos], nBufSize, 0);
                     vRecv.resize(nPos + max(nBytes, 0));
-                    if (nBytes == 0)
+                    if (nBytes > 0)
+                    {
+                        pnode->nLastRecv = GetTime();
+                    }
+                    else if (nBytes == 0)
                     {
                         // socket closed gracefully
                         if (!pnode->fDisconnect)
@@ -1127,6 +1131,7 @@ void ThreadSocketHandler2(void* parg)
                         if (nBytes > 0)
                         {
                             vSend.erase(vSend.begin(), vSend.begin() + nBytes);
+                            pnode->nLastSend = GetTime();
                         }
                         else if (nBytes == 0)
                         {
@@ -1140,6 +1145,42 @@ void ThreadSocketHandler2(void* parg)
                                 pnode->vSend.clear();
                         }
                     }
+                    if (vSend.empty())
+                        pnode->nLastSendEmpty = GetTime();
+                }
+            }
+
+            //
+            // Inactivity
+            //
+            // Dropping the connection is the whole cure: everything above this
+            // layer already knows how to reconnect, and a peer we cannot hear
+            // is worth exactly as much as no peer at all. Give a new
+            // connection a grace period first, or we would cut off peers that
+            // are still completing the rendezvous handshake.
+            //
+            if (GetTime() - pnode->nTimeConnected > BTF_HANDSHAKE_GRACE_SECS)
+            {
+                if (pnode->nLastRecv == 0 || pnode->nLastSend == 0)
+                {
+                    LogPrint("net", "socket no message in first %d seconds, recv=%d send=%d\n",
+                             BTF_HANDSHAKE_GRACE_SECS, pnode->nLastRecv != 0, pnode->nLastSend != 0);
+                    pnode->fDisconnect = true;
+                }
+                else if (GetTime() - pnode->nLastSend > BTF_SEND_STALL_SECS &&
+                         GetTime() - pnode->nLastSendEmpty > BTF_SEND_STALL_SECS)
+                {
+                    // We have had something queued to send for this long and
+                    // none of it has gone out: the socket accepts no more.
+                    LogPrint("net", "socket not sending\n");
+                    pnode->fDisconnect = true;
+                }
+                else if (GetTime() - pnode->nLastRecv > BTF_RECV_TIMEOUT_SECS)
+                {
+                    // The deaf case. Nothing has arrived for many block
+                    // intervals while the socket still looks perfectly fine.
+                    LogPrint("net", "socket inactivity timeout\n");
+                    pnode->fDisconnect = true;
                 }
             }
         }

src/net.h

@@ -51,6 +51,24 @@ static const int64        PEX_MIN_INTERVAL         = 60;
 // traffic is nothing: one per relay per five minutes.
 static const int          BTF_RENDEZVOUS_WAIT_SECS = 300;
 
+// Liveness of an established peer connection. BTF_RENDEZVOUS_WAIT_SECS bounds
+// the wait for a dial to arrive; these bound the connection that comes out of
+// it, which has the same failure mode once it goes quiet.
+//
+// A healthy peer relays a block inventory roughly every nTargetSpacing (two
+// minutes), so silence measured in block intervals is the honest signal.
+// Bitcoin used ninety minutes against ten-minute blocks -- nine intervals --
+// and the same nine intervals here is eighteen minutes; thirty leaves margin
+// for a slow stretch without letting a dead path last an hour.
+static const int          BTF_RECV_TIMEOUT_SECS    = 30 * 60;
+static const int          BTF_SEND_STALL_SECS      = 10 * 60;
+static const int          BTF_HANDSHAKE_GRACE_SECS = 60;
+
+// Send a ping after this long with nothing to say, so a peer running the
+// inactivity check above does not mistake a quiet node for a dead one. Must
+// stay well under BTF_RECV_TIMEOUT_SECS.
+static const int          BTF_PING_INTERVAL_SECS   = 10 * 60;
+
 // Descriptors to hand a peer: ours first, then peers that answered us.
 void BtfPexCollect(std::vector<std::string>& vDescOut);
 // Verify descriptors a peer sent and remember the good ones. Returns how many.
@@ -500,6 +518,16 @@ class CNode
     // Last "btfpeers" we accepted from this node, to rate-limit the exchange.
     int64 nLastPexRecv;
 
+    // Liveness. A peer whose network path dies silently -- NAT drops an idle
+    // mapping, a relay restarts, a route changes -- never sends FIN, so the
+    // socket stays readable-never and the node simply stops hearing from it
+    // with no error anywhere. These stamps are the only way to tell that
+    // apart from a peer that merely has nothing to say.
+    int64 nTimeConnected;
+    int64 nLastSend;
+    int64 nLastRecv;
+    int64 nLastSendEmpty;
+
 
     CNode(SOCKET hSocketIn, CAddress addrIn, bool fInboundIn=false)
     {
@@ -517,6 +545,10 @@ class CNode
         nRefCount = 0;
         nReleaseTime = 0;
         nLastPexRecv = 0;
+        nTimeConnected = GetTime();
+        nLastSend = 0;
+        nLastRecv = 0;
+        nLastSendEmpty = GetTime();
         vfSubscribe.assign(256, false);
 
         // Push a version message