Skip to content

Fix operationMode stuck in DECOMMISSION_FAILED during resumed decommission #4920

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
minal-kyada wants to merge 1 commit into
base: trunk
Choose a base branch
from
+45 −1
Open

Fix operationMode stuck in DECOMMISSION_FAILED during resumed decommission #4920

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ static void decommission(boolean shutdownNetworking, boolean force)
else if (InProgressSequences.isLeave(inProgress))
{
logger.info("Resuming decommission @ {} (current epoch = {}): {}", inProgress.latestModification, metadata.epoch, inProgress.status());
StorageService.instance.clearTransientMode();
}
else
{
Expand Down
5 changes: 4 additions & 1 deletion test/distributed/org/apache/cassandra/distributed/test/DecommissionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.io.IOException;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;

import net.bytebuddy.ByteBuddy;
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
Expand Down Expand Up @@ -141,10 +142,12 @@ public void testDecommissionAfterNodeRestart() throws Throwable

public static class BB
{
static final AtomicBoolean injected = new AtomicBoolean(false);
public static void install(ClassLoader classLoader, Integer num)
{
if (num == 2)
{
injected.set(false);
new ByteBuddy().rebase(UnbootstrapStreams.class)
.method(named("execute"))
.intercept(MethodDelegation.to(DecommissionTest.BB.class))
Expand All @@ -157,7 +160,7 @@ public static void install(ClassLoader classLoader, Integer num)
public static void execute(NodeId leaving, PlacementDeltas startLeave, PlacementDeltas midLeave, PlacementDeltas finishLeave,
@SuperCall Callable<?> zuper) throws ExecutionException, InterruptedException
{
if (!StorageService.instance.isDecommissionFailed())
if (injected.compareAndSet(false, true))
throw new ExecutionException(new RuntimeException("simulated error in prepareUnbootstrapStreaming"));

try
Expand Down
40 changes: 40 additions & 0 deletions test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,19 @@
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeVersion;
import org.apache.cassandra.tcm.transformations.PrepareLeave;
import org.apache.cassandra.tcm.transformations.Startup;
import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.FBUtilities;

import static net.bytebuddy.matcher.ElementMatchers.named;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
import static org.apache.cassandra.distributed.shared.ClusterUtils.pauseBeforeCommit;
import static org.apache.cassandra.distributed.shared.ClusterUtils.unpauseCommits;
import static org.apache.cassandra.distributed.shared.NetworkTopology.dcAndRack;
import static org.apache.cassandra.distributed.shared.NetworkTopology.networkTopology;
import static org.apache.cassandra.distributed.test.ring.BootstrapTest.populate;
Expand All @@ -84,6 +88,42 @@ public void testResumableDecom() throws IOException
}
}

@Test
public void testOperationModeOnDecomResume() throws Exception
{
// Node 2's first decomission attempt fails mid-stream (injected by BB), leaving it in
// DECOMISSION_FAILED. On resume, operationMode() must transition back to LEAVING before
// MID_LEAVE is committed. We pause the CMS just before that commit to assert the mode
// in the window where streaming has finished but the epoch has not yet advanced.
try (Cluster cluster = builder().withNodes(3)
.withConfig(config -> config.with(NETWORK, GOSSIP))
.withInstanceInitializer(BB::install)
.start())
{
populate(cluster, 0, 100, 1, 2, ConsistencyLevel.QUORUM);

IInvokableInstance cmsNode = cluster.get(1);
IInvokableInstance leavingNode = cluster.get(2);

leavingNode.nodetoolResult("decommission", "--force").asserts().failure();
leavingNode.runOnInstance(() -> assertEquals(StorageService.Mode.DECOMMISSION_FAILED, StorageService.instance.operationMode()));

Callable<Epoch> midLeavePaused = pauseBeforeCommit(cmsNode, e -> e instanceof PrepareLeave.MidLeave);

Thread resumeThread = new Thread(() -> leavingNode.nodetoolResult("decommission", "--force").asserts().success());
resumeThread.start();
midLeavePaused.call();

leavingNode.runOnInstance(() ->
assertEquals("operationMode during resumed decommission streaming should be LEAVING, not DECOMMISSION_FAILED",
StorageService.Mode.LEAVING,
StorageService.instance.operationMode()));

unpauseCommits(cmsNode);
resumeThread.join(TimeUnit.MINUTES.toMillis(2));
}
}

@Test
public void testAddressReuseAfterDecommission() throws IOException, ExecutionException, InterruptedException
{
Expand Down