From c7d5e762fb20e03690d66174a25ecd49e257c987 Mon Sep 17 00:00:00 2001 From: "zhong.zhou" Date: Sat, 4 Jul 2026 17:42:34 +0800 Subject: [PATCH] [storage]: avoid kvm rbd cli for ceph luks size Resolves: ZSV-12469 Change-Id: I5ed4f9f00e700e4818a5c440b6dc5b04edee9c79 --- .../ceph/primary/CephPrimaryStorageBase.java | 217 ++++++++++++++++-- 1 file changed, 192 insertions(+), 25 deletions(-) diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java index 47ffbda6611..e6eb650e1a4 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java @@ -90,6 +90,7 @@ import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import java.util.stream.Collectors; import static org.zstack.core.Platform.*; @@ -482,12 +483,31 @@ public static class KVMHostEncryptInPlaceCmd implements Serializable { @NoLogging public String encryptedDek; public String installPath; + public String targetInstallPath; } public static class KVMHostLuksRsp extends KVMAgentCommands.AgentResponse { public Long actualSize; } + public static class ReplaceLuksRbdCmd extends AgentCommand { + public String installPath; + public String targetInstallPath; + public String temporaryInstallPath; + public String sourceTrashInstallPath; + public boolean deleteSourceTrash; + } + + public static class ReplaceLuksRbdRsp extends AgentResponse { + public Long size; + public Long actualSize; + } + + public static class SwapInPlaceLuksRbdCmd extends AgentCommand { + public String installPath; + public String temporaryInstallPath; + } + public static class DeleteCmd extends AgentCommand { String installPath; long expirationTime; @@ -1434,6 +1454,8 @@ public int compareTo(SnapInfo snapInfo) { public static final String KVM_HOST_LUKS_ENCRYPT_IN_PLACE_PATH = "/ceph/primarystorage/kvmhost/encryptinplace"; public static final String KVM_HOST_LUKS_RESIZE_PATH = "/ceph/primarystorage/kvmhost/luksresize"; public static final String KVM_HOST_LUKS_CONVERT_PATH = "/ceph/primarystorage/kvmhost/luksconvert"; + public static final String LUKS_REPLACE_VOLUME_PATH = "/ceph/primarystorage/volume/luksreplace"; + public static final String LUKS_SWAP_IN_PLACE_PATH = "/ceph/primarystorage/volume/luksswapinplace"; public static final String KVM_HOST_IMAGESTORE_ENCRYPTED_DOWNLOAD_PATH = "/ceph/primarystorage/kvmhost/imagestore/encrypteddownload"; public static final String FLATTEN_PATH = "/ceph/primarystorage/volume/flatten"; public static final String SFTP_DOWNLOAD_PATH = "/ceph/primarystorage/sftpbackupstorage/download"; @@ -1854,9 +1876,21 @@ public void success(KVMHostLuksRsp ret) { // assumes the agent returned a bare pool/uuid relative path). vol.setInstallPath(installPath); vol.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); - vol.setActualSize(ret.actualSize); - reply.setVolume(vol); - bus.reply(msg, reply); + getRbdActualSizeFromPrimaryStorageAgent(volumeUuid, installPath, + new ReturnValueCompletion(msg) { + @Override + public void success(Long actualSize) { + vol.setActualSize(actualSize); + reply.setVolume(vol); + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); } }); return; @@ -2380,11 +2414,12 @@ private void createFromVolumeSnapshot(FlowTrigger trigger) { new ReturnValueCompletion(trigger) { @Override public void success(KVMHostLuksRsp rsp) { - if (rsp.actualSize != null) { - actualSize = rsp.actualSize; - } - cachePath = dstPath; - trigger.next(); + continueFlowWithRbdActualSize(image.getInventory().getUuid(), dstPath, trigger, returnValue -> { + if (returnValue != null) { + actualSize = returnValue; + } + cachePath = dstPath; + }); } @Override @@ -2791,8 +2826,8 @@ public void fail(ErrorCode err) { @Override public void success(KVMHostLuksRsp ret) { - actualSize = ret.actualSize; - trigger.next(); + continueFlowWithRbdActualSize(msg.getVolume().getUuid(), volumePath, trigger, + returnValue -> actualSize = returnValue); } }); return; @@ -2882,8 +2917,8 @@ public void fail(ErrorCode err) { @Override public void success(KVMHostLuksRsp ret) { - actualSize = ret.actualSize; - trigger.next(); + continueFlowWithRbdActualSize(msg.getVolume().getUuid(), volumePath, trigger, + returnValue -> actualSize = returnValue); } }); } @@ -3519,19 +3554,37 @@ private void handle(final EncryptVolumeBitsOnPrimaryStorageMsg msg) { KVMHostEncryptInPlaceCmd kcmd = new KVMHostEncryptInPlaceCmd(); kcmd.psUuid = self.getUuid(); kcmd.installPath = msg.getInstallPath(); + kcmd.targetInstallPath = makeTemporaryLuksRbdInstallPath(msg.getInstallPath(), "encrypting"); kcmd.encryptedDek = msg.getEncryptedDek(); httpCallToKvmHost(msg.getHostUuid(), KVM_HOST_LUKS_ENCRYPT_IN_PLACE_PATH, kcmd, KVMHostLuksRsp.class, new ReturnValueCompletion(msg) { @Override public void fail(ErrorCode err) { + deleteRbdBitsBestEffort(kcmd.targetInstallPath); reply.setError(err); bus.reply(msg, reply); } @Override public void success(KVMHostLuksRsp ret) { - bus.reply(msg, reply); + SwapInPlaceLuksRbdCmd rcmd = new SwapInPlaceLuksRbdCmd(); + rcmd.installPath = msg.getInstallPath(); + rcmd.temporaryInstallPath = kcmd.targetInstallPath; + httpCall(LUKS_SWAP_IN_PLACE_PATH, rcmd, AgentResponse.class, + new ReturnValueCompletion(msg) { + @Override + public void success(AgentResponse returnValue) { + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + deleteRbdBitsBestEffort(kcmd.targetInstallPath); + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); } }); } @@ -3576,7 +3629,7 @@ protected void handle(ConvertVolumeEncryptionOnPrimaryStorageMsg msg) { KVMHostLuksConvertCmd kcmd = new KVMHostLuksConvertCmd(); kcmd.psUuid = self.getUuid(); kcmd.installPath = item.getSourceInstallPath(); - kcmd.targetInstallPath = item.getTargetInstallPath(); + kcmd.targetInstallPath = makeTemporaryLuksRbdInstallPath(item.getTargetInstallPath(), "converting"); kcmd.sourceTrashInstallPath = item.getSourceTrashInstallPath(); kcmd.targetEncrypted = msg.isTargetEncrypted(); kcmd.virtualSize = msg.getVolume().getSize(); @@ -3586,16 +3639,36 @@ protected void handle(ConvertVolumeEncryptionOnPrimaryStorageMsg msg) { new ReturnValueCompletion(msg) { @Override public void fail(ErrorCode err) { + deleteRbdBitsBestEffort(kcmd.targetInstallPath); reply.setError(err); bus.reply(msg, reply); } @Override public void success(KVMHostLuksRsp ret) { - if (ret.actualSize != null) { - reply.setActualSizes(Collections.singletonMap(msg.getVolume().getUuid(), ret.actualSize)); - } - bus.reply(msg, reply); + ReplaceLuksRbdCmd rcmd = new ReplaceLuksRbdCmd(); + rcmd.installPath = item.getSourceInstallPath(); + rcmd.targetInstallPath = item.getTargetInstallPath(); + rcmd.temporaryInstallPath = kcmd.targetInstallPath; + rcmd.sourceTrashInstallPath = item.getSourceTrashInstallPath(); + rcmd.deleteSourceTrash = false; + httpCall(LUKS_REPLACE_VOLUME_PATH, rcmd, ReplaceLuksRbdRsp.class, + new ReturnValueCompletion(msg) { + @Override + public void success(ReplaceLuksRbdRsp rsp) { + if (rsp.actualSize != null) { + reply.setActualSizes(Collections.singletonMap(msg.getVolume().getUuid(), rsp.actualSize)); + } + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + deleteRbdBitsBestEffort(kcmd.targetInstallPath); + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); } }); } @@ -3784,6 +3857,31 @@ protected void httpCall(final String path, final Agent new HttpCaller<>(path, cmd, retClass, callback, unit, timeout).call(); } + private String makeTemporaryLuksRbdInstallPath(String installPath, String marker) { + return String.format("%s-%s-%s", installPath, marker, Platform.getUuid().substring(0, 8)); + } + + private void deleteRbdBitsBestEffort(String installPath) { + if (StringUtils.isBlank(installPath)) { + return; + } + + DeleteCmd cmd = new DeleteCmd(); + cmd.installPath = installPath; + httpCall(DELETE_PATH, cmd, DeleteRsp.class, new ReturnValueCompletion(new NopeCompletion()) { + @Override + public void success(DeleteRsp returnValue) { + logger.debug(String.format("deleted RBD image after failed LUKS operation: %s", installPath)); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format("failed to delete RBD image after failed LUKS operation: %s, because: %s", + installPath, errorCode)); + } + }); + } + /** * Send an HTTP cmd to a KVM host's kvmagent (not to cephagent). Used by the * LUKS variants of clone / createempty / encrypt-in-place, where qemu-img @@ -3827,6 +3925,39 @@ public void run(MessageReply reply) { }); } + private void getRbdActualSizeFromPrimaryStorageAgent(String resourceUuid, String installPath, + ReturnValueCompletion completion) { + syncVolumeSize(resourceUuid, installPath, new ReturnValueCompletion(completion) { + @Override + public void success(GetVolumeSizeRsp rsp) { + completion.success(rsp.actualSize); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format("failed to get RBD actual size from primary storage agent, resource[uuid:%s], installPath[%s], because: %s", + resourceUuid, installPath, errorCode)); + completion.success(null); + } + }); + } + + private void continueFlowWithRbdActualSize(String resourceUuid, String installPath, FlowTrigger trigger, + Consumer consumer) { + getRbdActualSizeFromPrimaryStorageAgent(resourceUuid, installPath, new ReturnValueCompletion(trigger) { + @Override + public void success(Long actualSize) { + consumer.accept(actualSize); + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + } + protected void resizeEncryptedRbdVolumeOnKvmHost(VolumeInventory volume, long size, ReturnValueCompletion completion) { String hostUuid = findConnectedHostForCephLuks(); @@ -3836,7 +3967,30 @@ protected void resizeEncryptedRbdVolumeOnKvmHost(VolumeInventory volume, long si cmd.encryptedDek = volumeEncryptedSecretHelper.materializeAndSealVolumeDekForHost(hostUuid, volume.getUuid()); cmd.virtualSize = size; - httpCallToKvmHost(hostUuid, KVM_HOST_LUKS_RESIZE_PATH, cmd, KVMHostLuksRsp.class, completion); + httpCallToKvmHost(hostUuid, KVM_HOST_LUKS_RESIZE_PATH, cmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(completion) { + @Override + public void success(KVMHostLuksRsp rsp) { + getRbdActualSizeFromPrimaryStorageAgent(volume.getUuid(), volume.getInstallPath(), + new ReturnValueCompletion(completion) { + @Override + public void success(Long actualSize) { + rsp.actualSize = actualSize; + completion.success(rsp); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); } public class HttpCaller { @@ -5578,12 +5732,25 @@ private void flattenSnapshotToEncryptedVolume(final CreateVolumeFromVolumeSnapsh new ReturnValueCompletion(completion) { @Override public void success(KVMHostLuksRsp rsp) { - reply.setInstallPath(volPath); - long asize = rsp.actualSize == null ? 1 : rsp.actualSize; - reply.setActualSize(asize); - reply.setSize(volume == null ? sp.getSize() : volume.getSize()); - bus.reply(msg, reply); - completion.done(); + getRbdActualSizeFromPrimaryStorageAgent(msg.getVolumeUuid(), volPath, + new ReturnValueCompletion(completion) { + @Override + public void success(Long actualSize) { + reply.setInstallPath(volPath); + long asize = actualSize == null ? 1 : actualSize; + reply.setActualSize(asize); + reply.setSize(volume == null ? sp.getSize() : volume.getSize()); + bus.reply(msg, reply); + completion.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + completion.done(); + } + }); } @Override