Skip to content

<fix>[flat-eip]: reduce EIP migration downtime #4693

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zstack-robot-2 wants to merge 1 commit into
base: feature-4.8.26-flat-eip
Choose a base branch
from
Open

<fix>[flat-eip]: reduce EIP migration downtime #4693

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 22 additions & 13 deletions compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6729,25 +6729,35 @@ public void done() {
@Override
public void handle(final Map data) {
VmInstanceInventory vm = VmInstanceInventory.valueOf(self);
extEmitter.afterMigrateVm(vm, vm.getLastHostUuid());
completion.success();
extEmitter.afterMigrateVm(vm, vm.getLastHostUuid(), new NoErrorCompletion(completion) {
@Override
public void done() {
completion.success();
}
});
}
}).error(new FlowErrorHandler(completion) {
@Override
public void handle(final ErrorCode errCode, Map data) {
String destHostUuid = spec.getDestHost().getUuid().equals(lastHostUuid) ? null : spec.getDestHost().getUuid();
extEmitter.failedToMigrateVm(VmInstanceInventory.valueOf(self), destHostUuid, errCode);
if (HostErrors.FAILED_TO_MIGRATE_VM_ON_HYPERVISOR.isEqual(errCode.getCode())) {
checkState(originalCopy.getHostUuid(), new NoErrorCompletion(completion) {
@Override
public void done() {
extEmitter.failedToMigrateVm(VmInstanceInventory.valueOf(self), destHostUuid, errCode,
new NoErrorCompletion(completion) {
@Override
public void done() {
if (!HostErrors.FAILED_TO_MIGRATE_VM_ON_HYPERVISOR.isEqual(errCode.getCode())) {
changeVmStateInDb(originState.getDrivenEvent());
completion.fail(errCode);
return;
}
});
} else {
changeVmStateInDb(originState.getDrivenEvent());
completion.fail(errCode);
}

checkState(originalCopy.getHostUuid(), new NoErrorCompletion(completion) {
@Override
public void done() {
completion.fail(errCode);
}
});
}
});
}
}).start();
}
Expand Down Expand Up @@ -8728,4 +8738,3 @@ public void run(MessageReply reply) {
});
}
}

55 changes: 45 additions & 10 deletions compute/src/main/java/org/zstack/compute/vm/VmInstanceExtensionPointEmitter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
package org.zstack.compute.vm;

import org.springframework.beans.factory.annotation.Autowired;
import org.zstack.core.asyncbatch.While;
import org.zstack.core.componentloader.PluginRegistry;
import org.zstack.core.errorcode.ErrorFacade;
import org.zstack.core.workflow.FlowChainBuilder;
import org.zstack.header.Component;
import org.zstack.header.core.Completion;
import org.zstack.header.core.NoErrorCompletion;
import org.zstack.header.core.WhileDoneCompletion;
import org.zstack.header.core.workflow.*;
import org.zstack.header.errorcode.ErrorCode;
import org.zstack.header.errorcode.ErrorCodeList;
import org.zstack.header.errorcode.SysErrors;
import org.zstack.header.vm.*;
import org.zstack.header.volume.VolumeInventory;
Expand Down Expand Up @@ -320,28 +324,59 @@ public void run(VmInstanceStartExtensionPoint arg) {
});
}

public void preMigrateVm(final VmInstanceInventory inv, final String dstHostUuid) {
CollectionUtils.safeForEach(migrateVmExtensions, arg -> arg.preMigrateVm(inv, dstHostUuid));
public void preMigrateVm(final VmInstanceInventory inv, final String dstHostUuid, Completion completion) {
new While<>(migrateVmExtensions).each((ext, comp) -> ext.preMigrateVm(inv, dstHostUuid, new Completion(comp) {
@Override
public void success() {
comp.done();
}

@Override
public void fail(ErrorCode errorCode) {
comp.addError(errorCode);
comp.allDone();
}
})).run(new WhileDoneCompletion(completion) {
@Override
public void done(ErrorCodeList errorCodeList) {
if (errorCodeList.getCauses().isEmpty()) {
completion.success();
} else {
completion.fail(errorCodeList.getCauses().get(0));
}
}
});
}

public void beforeMigrateVm(final VmInstanceInventory inv, final String dstHostUuid) {
CollectionUtils.safeForEach(migrateVmExtensions, arg -> arg.beforeMigrateVm(inv, dstHostUuid));
}

public void afterMigrateVm(final VmInstanceInventory inv, final String srcHostUuid) {
CollectionUtils.safeForEach(migrateVmExtensions, new ForEachFunction<VmInstanceMigrateExtensionPoint>() {
public void afterMigrateVm(final VmInstanceInventory inv, final String srcHostUuid, NoErrorCompletion completion) {
new While<>(migrateVmExtensions).each((ext, comp) -> ext.afterMigrateVm(inv, srcHostUuid, new NoErrorCompletion(comp) {
@Override
public void done() {
comp.done();
}
})).run(new WhileDoneCompletion(completion) {
@Override
public void run(VmInstanceMigrateExtensionPoint arg) {
arg.afterMigrateVm(inv, srcHostUuid);
public void done(ErrorCodeList errorCodeList) {
completion.done();
}
});
}

public void failedToMigrateVm(final VmInstanceInventory inv, final String dstHostUuid, final ErrorCode reason) {
CollectionUtils.safeForEach(migrateVmExtensions, new ForEachFunction<VmInstanceMigrateExtensionPoint>() {
public void failedToMigrateVm(final VmInstanceInventory inv, final String dstHostUuid, final ErrorCode reason,
NoErrorCompletion completion) {
new While<>(migrateVmExtensions).each((ext, comp) -> ext.failedToMigrateVm(inv, dstHostUuid, reason, new NoErrorCompletion(comp) {
@Override
public void done() {
comp.done();
}
})).run(new WhileDoneCompletion(completion) {
@Override
public void run(final VmInstanceMigrateExtensionPoint arg) {
arg.failedToMigrateVm(inv, dstHostUuid, reason);
public void done(ErrorCodeList errorCodeList) {
completion.done();
}
});
}
Expand Down
31 changes: 10 additions & 21 deletions compute/src/main/java/org/zstack/compute/vm/VmMigrateCallExtensionFlow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,13 @@
import org.springframework.beans.factory.annotation.Autowire;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Configurable;
import org.zstack.core.componentloader.PluginRegistry;
import org.zstack.header.core.Completion;
import org.zstack.header.core.workflow.Flow;
import org.zstack.header.core.workflow.FlowRollback;
import org.zstack.header.core.workflow.FlowTrigger;
import org.zstack.header.host.HostInventory;
import org.zstack.header.vm.MigrateVmMessage;
import org.zstack.header.vm.VmInstanceConstant;
import org.zstack.header.vm.VmInstanceMigrateExtensionPoint;
import org.zstack.header.vm.VmInstanceSpec;
import org.zstack.utils.CollectionUtils;
import org.zstack.utils.function.ForEachFunction;

import java.util.Map;

Expand All @@ -22,32 +18,25 @@
@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE)
public class VmMigrateCallExtensionFlow implements Flow {
@Autowired
protected PluginRegistry pluginRgty;
private VmInstanceExtensionPointEmitter extEmitter;

@Override
public void run(FlowTrigger trigger, Map data) {
final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString());

boolean migrateFromDest = false;
if (spec.getMessage() instanceof MigrateVmMessage) {
migrateFromDest = ((MigrateVmMessage)spec.getMessage()).isMigrateFromDestination();
}

final boolean fromDest = migrateFromDest;

final HostInventory destHost = spec.getDestHost();
for (VmInstanceMigrateExtensionPoint ext : pluginRgty.getExtensionList(VmInstanceMigrateExtensionPoint.class)) {
ext.preMigrateVm(spec.getVmInventory(), destHost.getUuid());
}
extEmitter.preMigrateVm(spec.getVmInventory(), destHost.getUuid(), new Completion(trigger) {
@Override
public void success() {
extEmitter.beforeMigrateVm(spec.getVmInventory(), destHost.getUuid());
trigger.next();
}

CollectionUtils.safeForEach(pluginRgty.getExtensionList(VmInstanceMigrateExtensionPoint.class), new ForEachFunction<VmInstanceMigrateExtensionPoint>() {
@Override
public void run(VmInstanceMigrateExtensionPoint ext) {
ext.beforeMigrateVm(spec.getVmInventory(), destHost.getUuid());
public void fail(org.zstack.header.errorcode.ErrorCode errorCode) {
trigger.fail(errorCode);
}
});

trigger.next();
}

@Override
Expand Down
27 changes: 24 additions & 3 deletions header/src/main/java/org/zstack/header/vm/VmInstanceMigrateExtensionPoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,34 @@
package org.zstack.header.vm;

import org.zstack.header.core.Completion;
import org.zstack.header.core.NoErrorCompletion;
import org.zstack.header.errorcode.ErrorCode;

public interface VmInstanceMigrateExtensionPoint {
void preMigrateVm(VmInstanceInventory inv, String destHostUuid);
default void preMigrateVm(VmInstanceInventory inv, String destHostUuid) {
}

default void preMigrateVm(VmInstanceInventory inv, String destHostUuid, Completion completion) {
preMigrateVm(inv, destHostUuid);
completion.success();
}

void beforeMigrateVm(VmInstanceInventory inv, String destHostUuid);

void afterMigrateVm(VmInstanceInventory inv, String srcHostUuid);
default void afterMigrateVm(VmInstanceInventory inv, String srcHostUuid) {
}

default void afterMigrateVm(VmInstanceInventory inv, String srcHostUuid, NoErrorCompletion completion) {
afterMigrateVm(inv, srcHostUuid);
completion.done();
}

default void failedToMigrateVm(VmInstanceInventory inv, String destHostUuid, ErrorCode reason) {
}

void failedToMigrateVm(VmInstanceInventory inv, String destHostUuid, ErrorCode reason);
default void failedToMigrateVm(VmInstanceInventory inv, String destHostUuid, ErrorCode reason,
NoErrorCompletion completion) {
failedToMigrateVm(inv, destHostUuid, reason);
completion.done();
}
}
81 changes: 58 additions & 23 deletions plugin/flatNetworkProvider/src/main/java/org/zstack/network/service/flat/FlatEipBackend.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import org.zstack.core.errorcode.ErrorFacade;
import org.zstack.core.timeout.ApiTimeoutManager;
import org.zstack.header.core.Completion;
import org.zstack.header.core.NoErrorCompletion;
import org.zstack.header.core.NopeCompletion;
import org.zstack.header.core.workflow.Flow;
import org.zstack.header.core.workflow.FlowRollback;
Expand Down Expand Up @@ -120,6 +121,7 @@ public static class DeleteEipCmd extends AgentCmd {

public static class BatchApplyEipCmd extends AgentCmd {
public List<EipTO> eips;
public boolean prepare;
}

public static class BatchDeleteEipCmd extends AgentCmd {
Expand All @@ -132,7 +134,14 @@ public static class BatchDeleteEipCmd extends AgentCmd {
public static final String BATCH_DELETE_EIP_PATH = "/flatnetworkprovider/eip/batchdelete";

@Override
public void preMigrateVm(VmInstanceInventory inv, String destHostUuid) {
public void preMigrateVm(VmInstanceInventory inv, String destHostUuid, Completion completion) {
List<EipTO> eips = getEipsByVmUuid(inv.getUuid());
if (eips == null || eips.isEmpty()) {
completion.success();
return;
}

batchApplyEips(eips, destHostUuid, true, false, completion);
}

@Override
Expand All @@ -141,52 +150,67 @@ public void beforeMigrateVm(VmInstanceInventory inv, String destHostUuid) {
}

@Override
public void afterMigrateVm(final VmInstanceInventory inv, String srcHostUuid) {
public void afterMigrateVm(final VmInstanceInventory inv, String srcHostUuid, NoErrorCompletion completion) {
List<EipTO> eips = getEipsByVmUuid(inv.getUuid());
if (eips == null || eips.isEmpty()) {
completion.done();
return;
}

batchDeleteEips(eips, srcHostUuid, new Completion(null) {
batchApplyEips(eips, inv.getHostUuid(), new Completion(completion) {
@Override
public void success() {
batchApplyEips(eips, inv.getHostUuid(), new Completion(null) {
batchDeleteEips(eips, srcHostUuid, new Completion(completion) {
@Override
public void success() {
logger.warn(String.format("after migration, successfully applied EIPs[uuids:%s] to the vm[uuid:%s, name:%s] on the destination host[uuid:%s] after delete eip on src host[uuid:%s] succeeded",
eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), inv.getName(), inv.getHostUuid(), srcHostUuid));
completion.done();
}

@Override
public void fail(ErrorCode errorCode) {
logger.warn(String.format("after migration, failed to apply EIPs[uuids:%s] to the vm[uuid:%s, name:%s] on the destination host[uuid:%s] after delete eip on src host[uuid:%s] succeeded, %s",
eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), inv.getName(), inv.getHostUuid(), srcHostUuid, errorCode));
logger.warn(String.format("failed to delete EIPs[vips:%s] for migrated vm[uuid:%s] on source host[uuid:%s], %s",
eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), srcHostUuid, errorCode));
completion.done();
}
});
}

@Override
public void fail(ErrorCode errorCode) {
batchApplyEips(eips, inv.getHostUuid(), new Completion(null) {
@Override
public void success() {
logger.warn(String.format("after migration, successfully applied EIPs[uuids:%s] to the vm[uuid:%s, name:%s] on the destination host[uuid:%s] after delete eip on src host[uuid:%s] failed",
eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), inv.getName(), inv.getHostUuid(), srcHostUuid));
}

@Override
public void fail(ErrorCode errorCode) {
logger.warn(String.format("after migration, failed to apply EIPs[uuids:%s] to the vm[uuid:%s, name:%s] on the destination host[uuid:%s] after delete eip on src host[uuid:%s] failed, %s",
eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), inv.getName(), inv.getHostUuid(), srcHostUuid, errorCode));
}
});
logger.warn(String.format("failed to enable EIPs[vips:%s] for migrated vm[uuid:%s] on destination host[uuid:%s], keep source EIPs on host[uuid:%s], %s",
eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), inv.getHostUuid(), srcHostUuid, errorCode));
completion.done();
}
});
}

@Override
public void failedToMigrateVm(VmInstanceInventory inv, String destHostUuid, ErrorCode reason) {
public void failedToMigrateVm(VmInstanceInventory inv, String destHostUuid, ErrorCode reason,
NoErrorCompletion completion) {
List<EipTO> eips = getEipsByVmUuid(inv.getUuid());
if (eips == null || eips.isEmpty() || destHostUuid == null) {
completion.done();
return;
}

if (destHostUuid.equals(inv.getHostUuid())) {
afterMigrateVm(inv, inv.getLastHostUuid(), completion);
return;
}

batchDeleteEips(eips, destHostUuid, new Completion(completion) {
@Override
public void success() {
completion.done();
}

@Override
public void fail(ErrorCode errorCode) {
logger.warn(String.format("failed to clean prepared EIPs[vips:%s] for vm[uuid:%s] on destination host[uuid:%s], %s",
eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), destHostUuid, errorCode));
completion.done();
}
});
}

@Transactional(readOnly = true)
Expand Down Expand Up @@ -519,12 +543,18 @@ public void run(MessageReply reply) {
}

private void batchApplyEips(List<EipTO> eips, String hostUuid, final Completion completion) {
batchApplyEips(eips, hostUuid, false, completion);
batchApplyEips(eips, hostUuid, false, false, completion);
}

private void batchApplyEips(List<EipTO> eips, String hostUuid, boolean noHostStatusCheck, final Completion completion) {
batchApplyEips(eips, hostUuid, false, noHostStatusCheck, completion);
}

private void batchApplyEips(List<EipTO> eips, String hostUuid, boolean prepare, boolean noHostStatusCheck,
final Completion completion) {
BatchApplyEipCmd cmd = new BatchApplyEipCmd();
cmd.eips = eips;
cmd.prepare = prepare;

KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg();
msg.setCommand(cmd);
Expand All @@ -547,6 +577,11 @@ public void run(MessageReply reply) {
return;
}

if (prepare) {
completion.success();
return;
}

List<String> vipUuids = CollectionUtils.transformToList(eips, new Function<String, EipTO>() {
@Override
public String call(EipTO arg) {
Expand Down