feat: retry pending fault propagation to disconnected peers

.changesets/fault-propagation-retry.md

@@ -0,0 +1,7 @@
+release: patch
+summary: Retry pending FAULT order propagation to disconnected peers via check_transitions()
+
+When propagate_fault() fails to send the FAULT order (e.g. TCP connection not yet
+established), the target is marked pending. check_transitions() automatically
+retries pending targets on each call while the board remains faulted, until the
+send succeeds.

Inc/ST-LIB_HIGH/Protections/FaultController.hpp

@@ -92,6 +92,7 @@ class FaultController {
 #ifdef STLIB_ETH
         propagation_targets = {};
         propagation_target_count = 0;
+        last_retry_us = 0;
 #endif
     }
 
@@ -191,15 +192,19 @@ class FaultController {
 
 #ifdef STLIB_ETH
     static void propagate_fault();
+    static void retry_pending_fault_propagation();
     static void on_fault_order_received();
 
     struct FaultPropagationTarget {
         OrderProtocol* socket;
         Order* fault_order;
+        bool pending = false;
     };
     static constexpr size_t max_propagation_targets = 8;
+    static constexpr uint64_t propagation_retry_period_us = 100'000;
     static array<FaultPropagationTarget, max_propagation_targets> propagation_targets;
     static size_t propagation_target_count;
+    static uint64_t last_retry_us;
 #endif
 
     static RuntimeStorage runtime_storage;

Src/ST-LIB_HIGH/Protections/FaultController.cpp

@@ -1,6 +1,7 @@
 #include "ST-LIB_HIGH/Protections/FaultController.hpp"
 
 #include "HALAL/Services/Diagnostics/Diagnostics.hpp"
+#include "HALAL/Services/Time/Scheduler.hpp"
 
 namespace {
 
@@ -164,6 +165,15 @@ void FaultController::check_transitions() {
         return;
     }
     global_machine->check_transitions();
+#ifdef STLIB_ETH
+    if (faulted) {
+        const uint64_t now_us = Scheduler::get_global_tick();
+        if (now_us - last_retry_us >= propagation_retry_period_us) {
+            last_retry_us = now_us;
+            retry_pending_fault_propagation();
+        }
+    }
+#endif
 }
 
 void FaultController::publish_fault_diagnostic(const FaultCause& cause) {
@@ -175,6 +185,7 @@ void FaultController::publish_fault_diagnostic(const FaultCause& cause) {
 array<FaultController::FaultPropagationTarget, FaultController::max_propagation_targets>
     FaultController::propagation_targets{};
 size_t FaultController::propagation_target_count = 0;
+uint64_t FaultController::last_retry_us = 0;
 
 void FaultController::register_fault_propagation(OrderProtocol* socket, Order* fault_order) {
     if (socket == nullptr || fault_order == nullptr)
@@ -200,9 +211,31 @@ void FaultController::propagate_fault() {
     for (size_t i = 0; i < propagation_target_count; i++) {
         auto& target = propagation_targets[i];
         if (target.socket != nullptr && target.fault_order != nullptr) {
-            if (!target.socket->send_order(*target.fault_order)) {
+            target.pending = true;
+            if (target.socket->send_order(*target.fault_order)) {
+                target.pending = false;
+            } else {
                 Diagnostics::Hub::publish_runtime_warning(
-                    "FAULT order propagation failed",
+                    "FAULT order propagation failed, will retry",
+                    false,
+                    __LINE__,
+                    __func__,
+                    __FILE__
+                );
+                Diagnostics::Hub::flush_urgent();
+            }
+        }
+    }
+}
+
+void FaultController::retry_pending_fault_propagation() {
+    for (size_t i = 0; i < propagation_target_count; i++) {
+        auto& target = propagation_targets[i];
+        if (target.pending && target.socket != nullptr && target.fault_order != nullptr) {
+            if (target.socket->send_order(*target.fault_order)) {
+                target.pending = false;
+                Diagnostics::Hub::publish_runtime_info(
+                    "FAULT order propagation succeeded after retry",
                     false,
                     __LINE__,
                     __func__,

Tests/TestAccess.hpp

@@ -35,6 +35,7 @@ struct FaultController {
 #ifdef STLIB_ETH
         ::FaultController::propagation_targets = {};
         ::FaultController::propagation_target_count = 0;
+        ::FaultController::last_retry_us = 0;
 #endif
     }