From d5fbf2303018e35867737bff3c49c6b18ef8047d Mon Sep 17 00:00:00 2001 From: Minal Kyada Date: Thu, 9 Jul 2026 15:05:27 -0700 Subject: [PATCH] node-decomission-resume-path-fix --- .../tcm/sequences/SingleNodeSequences.java | 1 + .../distributed/test/DecommissionTest.java | 5 ++- .../test/ring/DecommissionTest.java | 40 +++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java b/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java index 313700aaafff..82a1059b6ee9 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java +++ b/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java @@ -92,6 +92,7 @@ static void decommission(boolean shutdownNetworking, boolean force) else if (InProgressSequences.isLeave(inProgress)) { logger.info("Resuming decommission @ {} (current epoch = {}): {}", inProgress.latestModification, metadata.epoch, inProgress.status()); + StorageService.instance.clearTransientMode(); } else { diff --git a/test/distributed/org/apache/cassandra/distributed/test/DecommissionTest.java b/test/distributed/org/apache/cassandra/distributed/test/DecommissionTest.java index ad0120513b47..c0c9cee96cc2 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/DecommissionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/DecommissionTest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import net.bytebuddy.ByteBuddy; import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; @@ -141,10 +142,12 @@ public void testDecommissionAfterNodeRestart() throws Throwable public static class BB { + static final AtomicBoolean injected = new AtomicBoolean(false); public static void install(ClassLoader classLoader, Integer num) { if (num == 2) { + injected.set(false); new ByteBuddy().rebase(UnbootstrapStreams.class) .method(named("execute")) .intercept(MethodDelegation.to(DecommissionTest.BB.class)) @@ -157,7 +160,7 @@ public static void install(ClassLoader classLoader, Integer num) public static void execute(NodeId leaving, PlacementDeltas startLeave, PlacementDeltas midLeave, PlacementDeltas finishLeave, @SuperCall Callable zuper) throws ExecutionException, InterruptedException { - if (!StorageService.instance.isDecommissionFailed()) + if (injected.compareAndSet(false, true)) throw new ExecutionException(new RuntimeException("simulated error in prepareUnbootstrapStreaming")); try diff --git a/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java b/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java index 307956828efc..b6a271085fa0 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java @@ -53,8 +53,10 @@ import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeVersion; +import org.apache.cassandra.tcm.transformations.PrepareLeave; import org.apache.cassandra.tcm.transformations.Startup; import org.apache.cassandra.utils.CassandraVersion; import org.apache.cassandra.utils.FBUtilities; @@ -62,6 +64,8 @@ import static net.bytebuddy.matcher.ElementMatchers.named; import static org.apache.cassandra.distributed.api.Feature.GOSSIP; import static org.apache.cassandra.distributed.api.Feature.NETWORK; +import static org.apache.cassandra.distributed.shared.ClusterUtils.pauseBeforeCommit; +import static org.apache.cassandra.distributed.shared.ClusterUtils.unpauseCommits; import static org.apache.cassandra.distributed.shared.NetworkTopology.dcAndRack; import static org.apache.cassandra.distributed.shared.NetworkTopology.networkTopology; import static org.apache.cassandra.distributed.test.ring.BootstrapTest.populate; @@ -84,6 +88,42 @@ public void testResumableDecom() throws IOException } } + @Test + public void testOperationModeOnDecomResume() throws Exception + { + // Node 2's first decomission attempt fails mid-stream (injected by BB), leaving it in + // DECOMISSION_FAILED. On resume, operationMode() must transition back to LEAVING before + // MID_LEAVE is committed. We pause the CMS just before that commit to assert the mode + // in the window where streaming has finished but the epoch has not yet advanced. + try (Cluster cluster = builder().withNodes(3) + .withConfig(config -> config.with(NETWORK, GOSSIP)) + .withInstanceInitializer(BB::install) + .start()) + { + populate(cluster, 0, 100, 1, 2, ConsistencyLevel.QUORUM); + + IInvokableInstance cmsNode = cluster.get(1); + IInvokableInstance leavingNode = cluster.get(2); + + leavingNode.nodetoolResult("decommission", "--force").asserts().failure(); + leavingNode.runOnInstance(() -> assertEquals(StorageService.Mode.DECOMMISSION_FAILED, StorageService.instance.operationMode())); + + Callable midLeavePaused = pauseBeforeCommit(cmsNode, e -> e instanceof PrepareLeave.MidLeave); + + Thread resumeThread = new Thread(() -> leavingNode.nodetoolResult("decommission", "--force").asserts().success()); + resumeThread.start(); + midLeavePaused.call(); + + leavingNode.runOnInstance(() -> + assertEquals("operationMode during resumed decommission streaming should be LEAVING, not DECOMMISSION_FAILED", + StorageService.Mode.LEAVING, + StorageService.instance.operationMode())); + + unpauseCommits(cmsNode); + resumeThread.join(TimeUnit.MINUTES.toMillis(2)); + } + } + @Test public void testAddressReuseAfterDecommission() throws IOException, ExecutionException, InterruptedException {