diff --git a/plugin/flatNetworkProvider/src/main/java/org/zstack/network/service/flat/FlatEipBackend.java b/plugin/flatNetworkProvider/src/main/java/org/zstack/network/service/flat/FlatEipBackend.java index 0500fb4e2ea..fa0d2fccdbb 100755 --- a/plugin/flatNetworkProvider/src/main/java/org/zstack/network/service/flat/FlatEipBackend.java +++ b/plugin/flatNetworkProvider/src/main/java/org/zstack/network/service/flat/FlatEipBackend.java @@ -13,6 +13,7 @@ import org.zstack.core.errorcode.ErrorFacade; import org.zstack.core.timeout.ApiTimeoutManager; import org.zstack.header.core.Completion; +import org.zstack.header.core.FutureCompletion; import org.zstack.header.core.NopeCompletion; import org.zstack.header.core.workflow.Flow; import org.zstack.header.core.workflow.FlowRollback; @@ -57,6 +58,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static java.util.Arrays.asList; @@ -120,6 +122,7 @@ public static class DeleteEipCmd extends AgentCmd { public static class BatchApplyEipCmd extends AgentCmd { public List eips; + public boolean prepare; } public static class BatchDeleteEipCmd extends AgentCmd { @@ -133,6 +136,17 @@ public static class BatchDeleteEipCmd extends AgentCmd { @Override public void preMigrateVm(VmInstanceInventory inv, String destHostUuid) { + List eips = getEipsByVmUuid(inv.getUuid()); + if (eips == null || eips.isEmpty()) { + return; + } + + FutureCompletion completion = new FutureCompletion(null); + batchApplyEips(eips, destHostUuid, true, false, completion); + completion.await(TimeUnit.MINUTES.toMillis(30)); + if (!completion.isSuccess()) { + throw new OperationFailureException(completion.getErrorCode()); + } } @Override @@ -147,46 +161,54 @@ public void afterMigrateVm(final VmInstanceInventory inv, String srcHostUuid) { return; } - batchDeleteEips(eips, srcHostUuid, new Completion(null) { + batchApplyEips(eips, inv.getHostUuid(), new Completion(null) { @Override public void success() { - batchApplyEips(eips, inv.getHostUuid(), new Completion(null) { + batchDeleteEips(eips, srcHostUuid, new Completion(null) { @Override public void success() { - logger.warn(String.format("after migration, successfully applied EIPs[uuids:%s] to the vm[uuid:%s, name:%s] on the destination host[uuid:%s] after delete eip on src host[uuid:%s] succeeded", - eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), inv.getName(), inv.getHostUuid(), srcHostUuid)); } @Override public void fail(ErrorCode errorCode) { - logger.warn(String.format("after migration, failed to apply EIPs[uuids:%s] to the vm[uuid:%s, name:%s] on the destination host[uuid:%s] after delete eip on src host[uuid:%s] succeeded, %s", - eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), inv.getName(), inv.getHostUuid(), srcHostUuid, errorCode)); + logger.warn(String.format("failed to delete EIPs[vips:%s] for migrated vm[uuid:%s] on source host[uuid:%s], %s", + eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), srcHostUuid, errorCode)); } }); } @Override public void fail(ErrorCode errorCode) { - batchApplyEips(eips, inv.getHostUuid(), new Completion(null) { - @Override - public void success() { - logger.warn(String.format("after migration, successfully applied EIPs[uuids:%s] to the vm[uuid:%s, name:%s] on the destination host[uuid:%s] after delete eip on src host[uuid:%s] failed", - eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), inv.getName(), inv.getHostUuid(), srcHostUuid)); - } - - @Override - public void fail(ErrorCode errorCode) { - logger.warn(String.format("after migration, failed to apply EIPs[uuids:%s] to the vm[uuid:%s, name:%s] on the destination host[uuid:%s] after delete eip on src host[uuid:%s] failed, %s", - eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), inv.getName(), inv.getHostUuid(), srcHostUuid, errorCode)); - } - }); + logger.warn(String.format("failed to enable EIPs[vips:%s] for migrated vm[uuid:%s] on destination host[uuid:%s], keep source EIPs on host[uuid:%s], %s", + eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), inv.getHostUuid(), srcHostUuid, errorCode)); } }); } @Override public void failedToMigrateVm(VmInstanceInventory inv, String destHostUuid, ErrorCode reason) { + List eips = getEipsByVmUuid(inv.getUuid()); + if (eips == null || eips.isEmpty() || destHostUuid == null) { + return; + } + if (HostErrors.FAILED_TO_MIGRATE_VM_ON_HYPERVISOR.isEqual(reason.getCode())) { + logger.warn(String.format("keep prepared EIPs[vips:%s] on destination host[uuid:%s] because vm[uuid:%s] placement is uncertain after migration failure", + eips.stream().map(e -> e.vip).collect(Collectors.toList()), destHostUuid, inv.getUuid())); + return; + } + + batchDeleteEips(eips, destHostUuid, new Completion(null) { + @Override + public void success() { + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format("failed to clean prepared EIPs[vips:%s] for vm[uuid:%s] on destination host[uuid:%s], %s", + eips.stream().map(e -> e.vip).collect(Collectors.toList()), inv.getUuid(), destHostUuid, errorCode)); + } + }); } @Transactional(readOnly = true) @@ -250,13 +272,22 @@ public void fail(ErrorCode errorCode) { } private void vmMigrateToAnotherHost(final FlowTrigger trigger) { - batchDeleteEips(eips, struct.getOriginalHostUuid(), new NopeCompletion()); - applyHostUuidForRollback = struct.getOriginalHostUuid(); batchApplyEips(eips, struct.getCurrentHostUuid(), new Completion(trigger) { @Override public void success() { releaseHostUuidForRollback = struct.getCurrentHostUuid(); - trigger.next(); + batchDeleteEips(eips, struct.getOriginalHostUuid(), new Completion(trigger) { + @Override + public void success() { + applyHostUuidForRollback = struct.getOriginalHostUuid(); + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); } @Override @@ -268,13 +299,22 @@ public void fail(ErrorCode errorCode) { private void vmRunningFromUnknownStateHostChanged(final FlowTrigger trigger) { - batchDeleteEips(eips, struct.getOriginalHostUuid(), new NopeCompletion()); - applyHostUuidForRollback = struct.getOriginalHostUuid(); batchApplyEips(eips, struct.getCurrentHostUuid(), new Completion(trigger) { @Override public void success() { releaseHostUuidForRollback = struct.getCurrentHostUuid(); - trigger.next(); + batchDeleteEips(eips, struct.getOriginalHostUuid(), new Completion(trigger) { + @Override + public void success() { + applyHostUuidForRollback = struct.getOriginalHostUuid(); + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); } @Override @@ -519,12 +559,18 @@ public void run(MessageReply reply) { } private void batchApplyEips(List eips, String hostUuid, final Completion completion) { - batchApplyEips(eips, hostUuid, false, completion); + batchApplyEips(eips, hostUuid, false, false, completion); } private void batchApplyEips(List eips, String hostUuid, boolean noHostStatusCheck, final Completion completion) { + batchApplyEips(eips, hostUuid, false, noHostStatusCheck, completion); + } + + private void batchApplyEips(List eips, String hostUuid, boolean prepare, boolean noHostStatusCheck, + final Completion completion) { BatchApplyEipCmd cmd = new BatchApplyEipCmd(); cmd.eips = eips; + cmd.prepare = prepare; KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); msg.setCommand(cmd); @@ -547,6 +593,11 @@ public void run(MessageReply reply) { return; } + if (prepare) { + completion.success(); + return; + } + List vipUuids = CollectionUtils.transformToList(eips, new Function() { @Override public String call(EipTO arg) { diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/flat/eip/StartFlatNetworkVmWithEipCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/flat/eip/StartFlatNetworkVmWithEipCase.groovy index ac5cfa42337..23e07e0972c 100644 --- a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/flat/eip/StartFlatNetworkVmWithEipCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/flat/eip/StartFlatNetworkVmWithEipCase.groovy @@ -245,15 +245,15 @@ class StartFlatNetworkVmWithEipCase extends SubCase { vmNicUuid = vm.vmNics[0].uuid } - boolean deleteEipOnSrcHostFailed = false + List migrationEipOperations = Collections.synchronizedList([]) env.afterSimulator(FlatEipBackend.BATCH_DELETE_EIP_PATH) { - deleteEipOnSrcHostFailed = true + migrationEipOperations.add("delete") throw new HttpError(403, "on purpose") } - boolean applyEipOnDestHostSuccessed = false env.afterSimulator(FlatEipBackend.BATCH_APPLY_EIP_PATH) { rsp, HttpEntity e -> - applyEipOnDestHostSuccessed = true + def cmd = json(e.body, FlatEipBackend.BatchApplyEipCmd.class) + migrationEipOperations.add(cmd.prepare ? "prepare" : "active") return rsp } @@ -263,13 +263,17 @@ class StartFlatNetworkVmWithEipCase extends SubCase { } retryInSecs { - assert deleteEipOnSrcHostFailed == true - assert applyEipOnDestHostSuccessed == true + int prepareIndex = migrationEipOperations.indexOf("prepare") + int activeIndex = migrationEipOperations.indexOf("active") + int deleteIndex = migrationEipOperations.indexOf("delete") + assert prepareIndex >= 0 : "EIP migration did not prepare destination EIP: operations=${migrationEipOperations}" + assert activeIndex > prepareIndex : "Destination EIP was not activated after prepare: operations=${migrationEipOperations}" + assert deleteIndex > activeIndex : "Source EIP was deleted before destination activation: operations=${migrationEipOperations}" } - applyEipOnDestHostSuccessed = false + migrationEipOperations.clear() env.afterSimulator(FlatEipBackend.BATCH_DELETE_EIP_PATH) { rsp, HttpEntity e -> - deleteEipOnSrcHostFailed = false + migrationEipOperations.add("delete") return rsp } @@ -279,8 +283,12 @@ class StartFlatNetworkVmWithEipCase extends SubCase { } retryInSecs { - assert deleteEipOnSrcHostFailed == false - assert applyEipOnDestHostSuccessed == true + int prepareIndex = migrationEipOperations.indexOf("prepare") + int activeIndex = migrationEipOperations.indexOf("active") + int deleteIndex = migrationEipOperations.indexOf("delete") + assert prepareIndex >= 0 : "Reverse migration did not prepare destination EIP: operations=${migrationEipOperations}" + assert activeIndex > prepareIndex : "Reverse migration did not activate destination after prepare: operations=${migrationEipOperations}" + assert deleteIndex > activeIndex : "Reverse migration deleted source before destination activation: operations=${migrationEipOperations}" } }