Apply blocks at production round entry

libraries/chain/controller.cpp

@@ -3473,7 +3473,12 @@ struct controller_impl {
    }
 
    bool is_head_descendant_of_pending_lib() const {
-      return fork_db_.is_descendant_of_pending_savanna_lib(chain_head.id());
+      // True if pending_savanna_lib is on the chain from chain_head back to fork_db root: pending_savanna_lib is an
+      // ancestor of chain_head, or is chain_head itself. Answered from the head's finality_core, which tracks the
+      // canonical block_ref at each height across [last_final, head). No fork_db walk or mutex required.
+      const auto [pending_id, pending_timestamp] = fork_db_.pending_savanna_lib();
+      if (chain_head.id() == pending_id) return true;
+      return chain_head.extends(pending_id);
    }
 
    void set_savanna_lib(const block_id_type& id, block_timestamp_type timestamp) {

libraries/chain/fork_database.cpp

@@ -89,7 +89,6 @@ namespace sysio::chain {
       void             remove_impl( block_num_type block_num );
       bsp_t            head_impl(include_root_t include_root) const;
       bool             set_pending_savanna_lib_impl(const block_id_type& id, block_timestamp_type timestamp);
-      bool             is_descendant_of_pending_savanna_lib_impl(const block_id_type& id) const;
       bool             is_descendant_of_impl(const block_id_type& ancestor, const block_id_type& descendant) const;
       branch_t         fetch_branch_impl( const block_id_type& h, uint32_t trim_after_block_num ) const;
       block_branch_t   fetch_block_branch_impl( const block_id_type& h, uint32_t trim_after_block_num ) const;
@@ -368,19 +367,6 @@ namespace sysio::chain {
       return false;
    }
 
-   template<class BSP>
-   bool fork_database_type<BSP>::is_descendant_of_pending_savanna_lib( const block_id_type& id ) const {
-      std::lock_guard g( my->mtx );
-      return my->is_descendant_of_pending_savanna_lib_impl(id);
-   }
-
-   template<class BSP>
-   bool fork_database_impl<BSP>::is_descendant_of_pending_savanna_lib_impl(const block_id_type& id) const {
-      if (pending_savanna_lib_id == id)
-         return true;
-      return is_descendant_of_impl(pending_savanna_lib_id, id);
-   }
-
    template<class BSP>
    bool fork_database_type<BSP>::is_descendant_of(const block_id_type& ancestor, const block_id_type& descendant) const {
       std::lock_guard g( my->mtx );

libraries/chain/include/sysio/chain/block_handle.hpp

@@ -38,6 +38,12 @@ struct block_handle {
    void write(const std::filesystem::path& state_file);
    bool read(const std::filesystem::path& state_file);
 
+   // Returns true if `id` is a strict ancestor of this block within the finality_core's tracking range
+   // (block_num in [last_final_block_num, current_block_num)). A block does not extend itself.
+   bool extends(const block_id_type& id) const {
+      return _bsp && _bsp->core.extends(id);
+   }
+
    // Returns true iff this block carries a strong QC whose target is not in `head_handle`'s ancestry. Under Savanna's
    // strong-vote locking, finalizers locked on the QC target cannot later vote on any branch that does not extend it,
    // so a head whose branch does not include the QC target can never be covered by a future QC; it is permanently
@@ -77,11 +83,6 @@ struct block_handle {
       // Head's branch and the QC target are incompatible; locked out.
       return true;
    }
-
-   // Returns true if `id` is in this block's ancestry (or is this block itself within the finality_core's tracking range).
-   bool extends(const block_id_type& id) const {
-      return _bsp && _bsp->core.extends(id);
-   }
 };
 
 } // namespace sysio::chain

libraries/chain/include/sysio/chain/fork_database.hpp

@@ -115,11 +115,6 @@ namespace sysio::chain {
       std::pair<block_id_type, block_timestamp_type> pending_savanna_lib() const;
       bool set_pending_savanna_lib( const block_id_type& id, block_timestamp_type timestamp );
 
-      /**
-       * @return true if id is built on top of pending savanna lib or id == pending_savanna_lib
-       */
-      bool is_descendant_of_pending_savanna_lib( const block_id_type& id ) const;
-
       /**
        * @param ancestor the id of a possible ancestor block
        * @param descendant the id of a possible descendant block

plugins/producer_plugin/src/producer_plugin.cpp

@@ -943,13 +943,14 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
       auto& chain = chain_plug->chain();
 
+      const block_handle fhead = chain.fork_db_head();
+
       // While producing our own block, normally defer applying incoming blocks to avoid disrupting
       // production mid-block. Exception: if the fork-database best head carries a strong QC for a
       // block not in our applied head's ancestry, our head's branch can no longer form a QC that
       // wins fork-choice -- continuing to produce on it is pointless and the resulting blocks would
       // be orphaned at the next fork switch. In that case fall through and apply blocks now.
       if (in_producing_mode()) {
-         const block_handle fhead = chain.fork_db_head();
          if (!fhead.locks_out_branch_of(chain.head())) {
             fc_ilog(_log, "producing, fork database head at: #{} id: {}",
                     fhead.block_num(), fhead.id());
@@ -963,7 +964,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       }
 
       // no reason to abort_block if we have nothing ready to process
-      if (chain.head().id() == chain.fork_db_head().id()) {
+      if (chain.head().id() == fhead.id()) {
          return {}; // nothing to do
       }
 
@@ -2249,13 +2250,27 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
 
       // producers need to be able to start producing on schedule, do not apply blocks as it might take a long time to apply
       // unless head not a child of pending lib, as there is no reason ever to produce on a branch that is not a child of pending lib
-      while (in_speculating_mode() || !chain.is_head_descendant_of_pending_lib()) {
+      // also apply when fork_db head is ahead of our applied head on the same chain -- producing on a stale head when the
+      // canonical chain has already moved on just orphans our blocks at the next fork switch (under Savanna fork choice
+      // by latest_qc_block_timestamp, the chain that finalized first wins regardless of who built locally)
+      auto fork_db_ahead_on_same_chain = [&]() {
+         return chain.fork_db_head().extends(head.id());
+      };
+      while (in_speculating_mode() || !chain.is_head_descendant_of_pending_lib() || fork_db_ahead_on_same_chain()) {
          if (is_configured_producer())
             schedule_delayed_production_loop(weak_from_this(), _pending_block_deadline); // interrupt apply_blocks at deadline
 
          auto result = apply_blocks();
-         if (result.status == controller::apply_blocks_result_t::status_t::complete && result.num_blocks_applied == 0)
+         if (result.num_blocks_applied == 0) {
+            // No progress: either nothing to apply (status complete), or apply was interrupted on a block that
+            // cannot complete within deadline (e.g., infinite trx). Exit the loop -- retrying would hit the same
+            // wall, blocking the main thread from servicing net_plugin and fork-choice updates that could deliver
+            // a better head. In producing mode, fall through to produce on the current head (a competing block at
+            // the same height, which fork choice can then prefer over the unapplyable one by timestamp).
+            if (in_speculating_mode() && result.status != controller::apply_blocks_result_t::status_t::complete)
+               return start_block_result::waiting_for_block;
             return start_block_result::succeeded;
+         }
 
          head = chain.head();
          if (head.block_num() == chain.get_pause_at_block_num())
@@ -2839,7 +2854,13 @@ void producer_plugin_impl::schedule_production_loop() {
       _timer.async_wait([this, cid = ++_timer_corelation_id](const boost::system::error_code& ec) {
          if (ec != boost::asio::error::operation_aborted && cid == _timer_corelation_id) {
             interrupt_transaction(controller::interrupt_t::all_trx);
-            app().executor().post(priority::high, exec_queue::read_write, [this]() {
+            // Recheck cid in the posted lambda: another schedule_* call may have bumped _timer_corelation_id between the
+            // timer firing and the post running. Same pattern as schedule_maybe_produce_block / schedule_delayed_production_loop.
+            app().executor().post(priority::high, exec_queue::read_write, [this, cid]() {
+               if (cid != _timer_corelation_id) {
+                  fc_dlog(_log, "Failed-start retry timer expired, skipping");
+                  return;
+               }
                schedule_production_loop();
             });
          }
@@ -2929,7 +2950,17 @@ void producer_plugin_impl::schedule_delayed_production_loop(const std::weak_ptr<
       _timer.async_wait([this, cid = ++_timer_corelation_id](const boost::system::error_code& ec) {
          if (ec != boost::asio::error::operation_aborted && cid == _timer_corelation_id) {
             interrupt_transaction(controller::interrupt_t::all_trx);
-            app().executor().post(priority::high, exec_queue::read_write, [this]() {
+            // Recheck cid inside the posted lambda: between the timer callback firing and the executor
+            // running this lambda, another schedule_* call may have bumped _timer_corelation_id (typically
+            // the schedule_maybe_produce_block invoked after the next start_block). If we ran
+            // schedule_production_loop unconditionally here, the inner schedule_delayed_production_loop
+            // call would bump cid again and starve the just-scheduled produce_block timer. Mirrors the
+            // pattern schedule_maybe_produce_block uses.
+            app().executor().post(priority::high, exec_queue::read_write, [this, cid]() {
+               if (cid != _timer_corelation_id) {
+                  fc_dlog(_log, "Speculative/Production Change timer expired, skipping");
+                  return;
+               }
                schedule_production_loop();
             });
          }