From b9d277ba52cba6d124adf38c0bdc5cb2a5b11451 Mon Sep 17 00:00:00 2001 From: "zhong.zhou" Date: Tue, 30 Jun 2026 11:29:39 +0800 Subject: [PATCH] [storage]: support zbs volume encryption Add ZBS encrypted volume handling for create, clone, snapshot restore, template consumption, resize, lifecycle, trash restore, image change, reimage, and VM clone flows. Move ZBS-specific orchestration into storage encrypt extensions while keeping the ZBS controller as a dispatcher. Tests: git diff --check. Resolves: ZSV-12579 Change-Id: I2676f28a78f00a85c85847494703ba0eb4708547 --- conf/springConfigXml/encrypt.xml | 14 + .../addon/primary/CreateVolumeSpec.java | 27 ++ .../primary/PrimaryStorageControllerSvc.java | 6 +- .../primary/ZbsVolumeEncryptionBackend.java | 25 ++ .../ZbsVolumeEncryptionExtensionPoint.java | 32 ++ .../EncryptVolumeBitsOnPrimaryStorageMsg.java | 18 +- ...ncryptVolumeBitsOnPrimaryStorageReply.java | 9 + .../zstack/expon/ExponStorageController.java | 12 +- .../storage/zbs/ZbsStorageController.java | 68 +++- .../zbs/ZbsVolumeEncryptionBackendImpl.java | 253 +++++++++++++++ .../addon/primary/ExternalPrimaryStorage.java | 63 +++- .../ZbsEncryptedEmptyVolumeCreator.java | 98 ++++++ .../ZbsEncryptedSnapshotVolumeCopier.java | 135 ++++++++ .../encrypt/ZbsEncryptedVolumeCloner.java | 305 ++++++++++++++++++ .../encrypt/ZbsEncryptedVolumeResizer.java | 96 ++++++ .../ZbsVolumeEncryptInPlaceHandler.java | 141 ++++++++ .../encrypt/ZbsVolumeEncryptionCleanup.java | 32 ++ .../encrypt/ZbsVolumeEncryptionCommands.java | 47 +++ .../encrypt/ZbsVolumeEncryptionConstants.java | 11 + .../encrypt/ZbsVolumeEncryptionExtension.java | 74 +++++ .../encrypt/ZbsVolumeEncryptionKvmCaller.java | 55 ++++ .../encrypt/ZbsVolumeEncryptionMaterial.java | 11 + .../ZbsVolumeEncryptionMaterialFactory.java | 83 +++++ .../volume/VolumeInPlaceEncryptor.java | 34 +- .../storage/volume/VolumeManagerImpl.java | 3 + .../VolumeManagerImplHostSelectionTest.groovy | 15 + .../ZbsVolumeEncryptionExtensionTest.groovy | 61 ++++ 27 files changed, 1705 insertions(+), 23 deletions(-) create mode 100644 header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionBackend.java create mode 100644 header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionExtensionPoint.java create mode 100644 plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsVolumeEncryptionBackendImpl.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedEmptyVolumeCreator.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedSnapshotVolumeCopier.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedVolumeCloner.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedVolumeResizer.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptInPlaceHandler.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionCleanup.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionCommands.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionConstants.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionExtension.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionKvmCaller.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionMaterial.java create mode 100644 storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionMaterialFactory.java create mode 100644 test/src/test/groovy/org/zstack/test/unittest/storage/encrypt/ZbsVolumeEncryptionExtensionTest.groovy diff --git a/conf/springConfigXml/encrypt.xml b/conf/springConfigXml/encrypt.xml index 4cac18b6b70..9e2515a8c84 100644 --- a/conf/springConfigXml/encrypt.xml +++ b/conf/springConfigXml/encrypt.xml @@ -42,6 +42,20 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/CreateVolumeSpec.java b/header/src/main/java/org/zstack/header/storage/addon/primary/CreateVolumeSpec.java index 81587db27c8..8be31aff4de 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/CreateVolumeSpec.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/CreateVolumeSpec.java @@ -7,6 +7,9 @@ public class CreateVolumeSpec { private String uuid; private long size; private VolumeQos qos; + private boolean encrypted; + private String hostUuid; + private String encryptedDek; private String allocatedUrl; private boolean dryRun; @@ -58,4 +61,28 @@ public String getName() { public void setName(String name) { this.name = name; } + + public boolean isEncrypted() { + return encrypted; + } + + public void setEncrypted(boolean encrypted) { + this.encrypted = encrypted; + } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getEncryptedDek() { + return encryptedDek; + } + + public void setEncryptedDek(String encryptedDek) { + this.encryptedDek = encryptedDek; + } } diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java index 5d6896960ce..a1269c93b53 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java @@ -4,7 +4,10 @@ import org.zstack.header.core.ReturnValueCompletion; import org.zstack.header.host.HostInventory; import org.zstack.header.storage.addon.*; +import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageReply; import org.zstack.header.storage.snapshot.VolumeSnapshotStats; +import org.zstack.header.volume.VolumeInventory; import org.zstack.header.volume.VolumeProtocol; import org.zstack.header.volume.VolumeStats; @@ -30,6 +33,7 @@ public interface PrimaryStorageControllerSvc { void trashVolume(String installPath, Completion comp); void cloneVolume(String srcInstallPath, CreateVolumeSpec dst, ReturnValueCompletioncomp); void copyVolume(String srcInstallPath, CreateVolumeSpec dst, ReturnValueCompletioncomp); + void encryptVolumeBits(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnValueCompletion comp); void flattenVolume(String installPath, ReturnValueCompletioncomp); // support uri or path @@ -37,7 +41,7 @@ public interface PrimaryStorageControllerSvc { void batchStats(Collection installPath, ReturnValueCompletion> comp); - void expandVolume(String installPath, long size, ReturnValueCompletion comp); + void expandVolume(VolumeInventory volume, long size, ReturnValueCompletion comp); void setVolumeQos(BaseVolumeInfo v, Completion comp); void deleteVolumeQos(BaseVolumeInfo v, Completion comp); void export(ExportSpec espec, VolumeProtocol protocol, ReturnValueCompletion comp); diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionBackend.java b/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionBackend.java new file mode 100644 index 00000000000..7ffeb6668c9 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionBackend.java @@ -0,0 +1,25 @@ +package org.zstack.header.storage.addon.primary; + +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.volume.VolumeStats; + +public interface ZbsVolumeEncryptionBackend { + String getPrimaryStorageUuid(); + + String buildConfiguredVolumePath(String volumeName); + + String buildEncryptedTargetPath(String installPath); + + void createLuksBackingVolume(String installPath, long virtualSize, ReturnValueCompletion completion); + + void deleteLuksBackingVolume(String installPath); + + void cloneVolumeAsBacking(String srcInstallPath, CreateVolumeSpec dst, ReturnValueCompletion completion); + + void resolveSnapshotPathForQemu(String installPath, ReturnValueCompletion completion); + + void checkNoSnapshots(String installPath, Completion completion); + + void stats(String installPath, ReturnValueCompletion completion); +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionExtensionPoint.java b/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionExtensionPoint.java new file mode 100644 index 00000000000..59fb0a28519 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionExtensionPoint.java @@ -0,0 +1,32 @@ +package org.zstack.header.storage.addon.primary; + +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageReply; +import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeStats; +import org.zstack.header.volume.VolumeVO; + +public interface ZbsVolumeEncryptionExtensionPoint { + String PRIMARY_STORAGE_IDENTITY = "zbs"; + + void createEncryptedEmptyVolume(ZbsVolumeEncryptionBackend backend, CreateVolumeSpec spec, + ReturnValueCompletion completion); + + void cloneEncryptedVolumeFromImage(ZbsVolumeEncryptionBackend backend, String srcInstallPath, CreateVolumeSpec dst, + ReturnValueCompletion completion); + + void copyEncryptedVolumeFromSnapshot(ZbsVolumeEncryptionBackend backend, String srcInstallPath, CreateVolumeSpec dst, + ReturnValueCompletion completion); + + void encryptVolumeBits(ZbsVolumeEncryptionBackend backend, EncryptVolumeBitsOnPrimaryStorageMsg msg, + ReturnValueCompletion completion); + + void resizeEncryptedVolume(String primaryStorageUuid, VolumeInventory volume, long size, + ReturnValueCompletion completion); + + void beforeTakeSnapshot(String primaryStorageUuid, VolumeVO volume, VolumeSnapshotInventory snapshot); + + void afterTakeSnapshot(String primaryStorageUuid, VolumeVO volume, VolumeSnapshotInventory snapshot); +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageMsg.java index d29677f8ec5..b03ddfc4c23 100644 --- a/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageMsg.java +++ b/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageMsg.java @@ -3,20 +3,12 @@ import org.zstack.header.log.NoLogging; import org.zstack.header.message.NeedReplyMessage; -/** - * Triggers an in-place LUKS encryption of an existing volume file on primary storage. - * Used after downloading a data-volume template's plain bits to LocalStorage when the - * volume is marked encrypted: the agent converts the plain qcow2/raw at {@link #installPath} - * into a LUKS-encrypted qcow2 (overwriting in place). - * - * The DEK is passed as a host-enveloped encryptedDek. The kvmagent unwraps it and - * creates any single-use secret material file locally as needed. - */ public class EncryptVolumeBitsOnPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage { private String primaryStorageUuid; private String hostUuid; private String volumeUuid; private String installPath; + private String targetInstallPath; @NoLogging private String encryptedDek; @@ -53,6 +45,14 @@ public void setInstallPath(String installPath) { this.installPath = installPath; } + public String getTargetInstallPath() { + return targetInstallPath; + } + + public void setTargetInstallPath(String targetInstallPath) { + this.targetInstallPath = targetInstallPath; + } + public String getEncryptedDek() { return encryptedDek; } diff --git a/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageReply.java b/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageReply.java index 2410ced31e6..481aa3f28f7 100644 --- a/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageReply.java +++ b/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageReply.java @@ -3,4 +3,13 @@ import org.zstack.header.message.MessageReply; public class EncryptVolumeBitsOnPrimaryStorageReply extends MessageReply { + private String installPath; + + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } } diff --git a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java index b7bd239ea6f..0f6cb4abfc2 100644 --- a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java +++ b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java @@ -27,6 +27,7 @@ import org.zstack.header.core.ReturnValueCompletion; import org.zstack.header.errorcode.ErrorCode; import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.errorcode.SysErrors; import org.zstack.header.expon.HealthStatus; import org.zstack.header.host.HostInventory; import org.zstack.header.host.HostVO; @@ -34,6 +35,8 @@ import org.zstack.header.image.ImageConstant; import org.zstack.header.storage.addon.*; import org.zstack.header.storage.addon.primary.*; +import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageReply; import org.zstack.header.storage.primary.ImageCacheInventory; import org.zstack.header.storage.primary.VolumeSnapshotCapability; import org.zstack.header.storage.snapshot.VolumeSnapshotStats; @@ -1093,6 +1096,12 @@ public void fail(ErrorCode errorCode) { }); } + @Override + public void encryptVolumeBits(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnValueCompletion comp) { + String details = String.format("primary storage controller[%s] does not support encrypting volume bits in place", getIdentity()); + comp.fail(new ErrorCode(SysErrors.UNIMPLEMENTED_OPERATION_ERROR.toString(), "unimplemented operation", details)); + } + @Override public void flattenVolume(String installPath, ReturnValueCompletion comp) { // TODO flatten snapshot @@ -1125,7 +1134,8 @@ public void batchStats(Collection installPath, ReturnValueCompletion comp) { + public void expandVolume(VolumeInventory volume, long size, ReturnValueCompletion comp) { + String installPath = volume.getInstallPath(); VolumeModule vol = apiHelper.expandVolume(getVolIdFromPath(installPath), size); VolumeStats stats = new VolumeStats(); stats.setInstallPath(installPath); diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index e2dab6b4eae..70a5420c544 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -12,6 +12,7 @@ import org.zstack.core.asyncbatch.While; import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.componentloader.PluginRegistry; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.SQL; import org.zstack.core.workflow.FlowChainBuilder; @@ -36,6 +37,7 @@ import org.zstack.header.storage.primary.*; import org.zstack.header.storage.snapshot.VolumeSnapshotStats; import org.zstack.header.volume.VolumeConstant; +import org.zstack.header.volume.VolumeInventory; import org.zstack.header.volume.VolumeProtocol; import org.zstack.header.volume.VolumeStats; import org.zstack.kvm.KVMHostAsyncHttpCallMsg; @@ -77,6 +79,8 @@ public class ZbsStorageController implements PrimaryStorageControllerSvc, Primar private ResourceConfigFacade rcf; @Autowired private CloudBus bus; + @Autowired + private PluginRegistry pluginRgty; private ExternalPrimaryStorageVO self; private AddonInfo addonInfo; @@ -581,10 +585,34 @@ private List getSelfPools() { return logicalPoolInfos; } + private ZbsVolumeEncryptionExtensionPoint getVolumeEncryptionExtension() { + List exts = pluginRgty.getExtensionList(ZbsVolumeEncryptionExtensionPoint.class); + if (exts.isEmpty()) { + throw new OperationFailureException(operr("cannot find ZBS volume encryption extension")); + } + return exts.get(0); + } + + private ZbsVolumeEncryptionBackend encryptionBackend() { + if (config == null || addonInfo == null) { + reloadDbInfo(); + } + return new ZbsVolumeEncryptionBackendImpl(this, self.getUuid(), addonInfo, config); + } + @Override public void createVolume(CreateVolumeSpec v, ReturnValueCompletion comp) { reloadDbInfo(); + if (v.isEncrypted()) { + try { + getVolumeEncryptionExtension().createEncryptedEmptyVolume(encryptionBackend(), v, comp); + } catch (OperationFailureException e) { + comp.fail(e.getErrorCode()); + } + return; + } + CreateVolumeCmd cmd = new CreateVolumeCmd(); cmd.setLogicalPool(config.getLogicalPoolName()); cmd.setVolume(v.getName()); @@ -627,6 +655,15 @@ public void trashVolume(String installPath, Completion comp) { @Override public void cloneVolume(String srcInstallPath, CreateVolumeSpec dst, ReturnValueCompletion comp) { + if (dst.isEncrypted()) { + try { + getVolumeEncryptionExtension().cloneEncryptedVolumeFromImage(encryptionBackend(), srcInstallPath, dst, comp); + } catch (OperationFailureException e) { + comp.fail(e.getErrorCode()); + } + return; + } + VolumeStats stats = new VolumeStats(); FlowChain chain = FlowChainBuilder.newShareFlowChain(); @@ -711,6 +748,15 @@ public void handle(ErrorCode errCode, Map data) { @Override public void copyVolume(String srcInstallPath, CreateVolumeSpec dst, ReturnValueCompletion comp) { + if (dst.isEncrypted()) { + try { + getVolumeEncryptionExtension().copyEncryptedVolumeFromSnapshot(encryptionBackend(), srcInstallPath, dst, comp); + } catch (OperationFailureException e) { + comp.fail(e.getErrorCode()); + } + return; + } + CopyCmd cmd = new CopyCmd(); cmd.setPath(srcInstallPath); cmd.setDstVolume(dst.getName()); @@ -734,6 +780,15 @@ public void fail(ErrorCode errorCode) { }); } + @Override + public void encryptVolumeBits(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnValueCompletion comp) { + try { + getVolumeEncryptionExtension().encryptVolumeBits(encryptionBackend(), msg, comp); + } catch (OperationFailureException e) { + comp.fail(e.getErrorCode()); + } + } + @Override public void flattenVolume(String installPath, ReturnValueCompletion comp) { FlattenVolumeCmd cmd = new FlattenVolumeCmd(); @@ -808,7 +863,18 @@ public void fail(ErrorCode errorCode) { } @Override - public void expandVolume(String installPath, long size, ReturnValueCompletion comp) { + public void expandVolume(VolumeInventory volume, long size, ReturnValueCompletion comp) { + if (Boolean.TRUE.equals(volume.getEncrypted())) { + List exts = pluginRgty.getExtensionList(ZbsVolumeEncryptionExtensionPoint.class); + if (exts.isEmpty()) { + comp.fail(operr("cannot find ZBS volume encryption extension")); + return; + } + exts.get(0).resizeEncryptedVolume(self.getUuid(), volume, size, comp); + return; + } + + String installPath = volume.getInstallPath(); ExpandVolumeCmd cmd = new ExpandVolumeCmd(); cmd.setPath(installPath); cmd.setUnit(getSizeUnit(addonInfo.getClusterInfo().getVersion())); diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsVolumeEncryptionBackendImpl.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsVolumeEncryptionBackendImpl.java new file mode 100644 index 00000000000..200b3165f49 --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsVolumeEncryptionBackendImpl.java @@ -0,0 +1,253 @@ +package org.zstack.storage.zbs; + +import org.apache.commons.lang.StringUtils; +import org.zstack.cbd.AddonInfo; +import org.zstack.cbd.Config; +import org.zstack.cbd.LogicalPoolInfo; +import org.zstack.header.core.Completion; +import org.zstack.header.core.NopeCompletion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.storage.addon.primary.CreateVolumeSpec; +import org.zstack.header.storage.addon.primary.ZbsVolumeEncryptionBackend; +import org.zstack.header.volume.VolumeConstant; +import org.zstack.header.volume.VolumeStats; + +import java.util.UUID; + +import static org.zstack.core.Platform.operr; +import static org.zstack.storage.zbs.ZbsHelper.alignSizeTo; +import static org.zstack.storage.zbs.ZbsHelper.buildVolumePath; +import static org.zstack.storage.zbs.ZbsHelper.getSizeUnit; + +class ZbsVolumeEncryptionBackendImpl implements ZbsVolumeEncryptionBackend { + private static final String QUERY_SNAPSHOT_PATH = "/zbs/primarystorage/snapshot/query"; + private static final long LUKS_PAYLOAD_OFFSET = 2068480L; + + private final ZbsStorageController controller; + private final String primaryStorageUuid; + private final AddonInfo addonInfo; + private final Config config; + + ZbsVolumeEncryptionBackendImpl(ZbsStorageController controller, String primaryStorageUuid, + AddonInfo addonInfo, Config config) { + this.controller = controller; + this.primaryStorageUuid = primaryStorageUuid; + this.addonInfo = addonInfo; + this.config = config; + } + + @Override + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + @Override + public String buildConfiguredVolumePath(String volumeName) { + String logicalPoolName = config.getLogicalPoolName(); + String physicalPoolName = addonInfo.getLogicalPoolInfos().stream() + .filter(it -> logicalPoolName.equals(it.getLogicalPoolName())) + .map(LogicalPoolInfo::getPhysicalPoolName) + .filter(StringUtils::isNotBlank) + .findFirst() + .orElse(logicalPoolName + "_physical"); + return buildVolumePath(physicalPoolName, logicalPoolName, volumeName); + } + + @Override + public String buildEncryptedTargetPath(String installPath) { + CbdPath path = parseActiveCbdPath(installPath); + String targetVolume = String.format("%s-encrypted-%s", path.volume, + UUID.randomUUID().toString().replace("-", "").substring(0, 8)); + return buildVolumePath(path.physicalPool, path.logicalPool, targetVolume); + } + + @Override + public void createLuksBackingVolume(String installPath, long virtualSize, ReturnValueCompletion completion) { + CbdPath path; + try { + path = parseActiveCbdPath(installPath); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + return; + } + + ZbsStorageController.CreateVolumeCmd cmd = new ZbsStorageController.CreateVolumeCmd(); + cmd.setLogicalPool(path.logicalPool); + cmd.setVolume(path.volume); + cmd.setUnit(getSizeUnit(addonInfo.getClusterInfo().getVersion())); + cmd.setSize(alignSizeTo(luksBackingSize(virtualSize), cmd.getUnit())); + cmd.setSkipIfExisting(false); + + controller.httpCall(ZbsStorageController.CREATE_VOLUME_PATH, cmd, ZbsStorageController.CreateVolumeRsp.class, + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsStorageController.CreateVolumeRsp returnValue) { + completion.success(returnValue.getInstallPath()); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + @Override + public void deleteLuksBackingVolume(String installPath) { + controller.doDeleteVolume(installPath, true, new NopeCompletion()); + } + + @Override + public void cloneVolumeAsBacking(String srcInstallPath, CreateVolumeSpec dst, + ReturnValueCompletion completion) { + ZbsStorageController.CloneVolumeCmd cmd = new ZbsStorageController.CloneVolumeCmd(); + cmd.setPath(srcInstallPath); + cmd.setDstVolume(dst.getName()); + + controller.httpCall(ZbsStorageController.CLONE_VOLUME_PATH, cmd, ZbsStorageController.CloneVolumeRsp.class, + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsStorageController.CloneVolumeRsp returnValue) { + VolumeStats stats = new VolumeStats(); + stats.setInstallPath(returnValue.getInstallPath()); + stats.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); + stats.setSize(returnValue.getSize()); + stats.setActualSize(returnValue.getActualSize()); + stats.setParentUri(srcInstallPath); + completion.success(stats); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + @Override + public void resolveSnapshotPathForQemu(String installPath, ReturnValueCompletion completion) { + if (!installPath.contains("@")) { + completion.success(installPath); + return; + } + + QuerySnapshotCmd cmd = new QuerySnapshotCmd(); + cmd.setPath(installPath); + controller.httpCall(QUERY_SNAPSHOT_PATH, cmd, QuerySnapshotRsp.class, + new ReturnValueCompletion(completion) { + @Override + public void success(QuerySnapshotRsp returnValue) { + completion.success(returnValue.getInstallPath()); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + @Override + public void checkNoSnapshots(String installPath, Completion completion) { + try { + parseActiveCbdPath(installPath); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + return; + } + + QuerySnapshotCmd cmd = new QuerySnapshotCmd(); + cmd.setPath(installPath); + controller.httpCall(QUERY_SNAPSHOT_PATH, cmd, QuerySnapshotRsp.class, + new ReturnValueCompletion(completion) { + @Override + public void success(QuerySnapshotRsp returnValue) { + if (returnValue.isHasSnapshot()) { + completion.fail(operr("cannot encrypt ZBS volume[%s] in place because it has snapshots", installPath)); + return; + } + completion.success(); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + @Override + public void stats(String installPath, ReturnValueCompletion completion) { + controller.stats(installPath, completion); + } + + private CbdPath parseCbdPath(String installPath) { + String path = installPath.startsWith("cbd:") ? installPath.substring("cbd:".length()) : installPath; + String[] parts = path.split("/", 3); + if (parts.length != 3) { + throw new OperationFailureException(operr("invalid ZBS CBD path[%s]", installPath)); + } + + CbdPath ret = new CbdPath(); + ret.physicalPool = parts[0]; + ret.logicalPool = parts[1]; + String[] volumeAndSnapshot = parts[2].split("@", 2); + ret.volume = volumeAndSnapshot[0]; + ret.snapshot = volumeAndSnapshot.length > 1 ? volumeAndSnapshot[1] : null; + return ret; + } + + private CbdPath parseActiveCbdPath(String installPath) { + CbdPath path = parseCbdPath(installPath); + if (StringUtils.isNotBlank(path.snapshot)) { + throw new OperationFailureException(operr("ZBS operation requires active volume path, but got[%s]", installPath)); + } + return path; + } + + private long luksBackingSize(long virtualSize) { + return virtualSize + LUKS_PAYLOAD_OFFSET; + } + + public static class QuerySnapshotRsp extends ZbsStorageController.AgentResponse { + private boolean hasSnapshot; + private String installPath; + + public boolean isHasSnapshot() { + return hasSnapshot; + } + + public void setHasSnapshot(boolean hasSnapshot) { + this.hasSnapshot = hasSnapshot; + } + + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } + } + + public static class QuerySnapshotCmd extends ZbsStorageController.AgentCommand { + private String path; + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + } + + private static class CbdPath { + private String physicalPool; + private String logicalPool; + private String volume; + private String snapshot; + } +} diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java index 12e0c3d3701..ed60c4a1f01 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java @@ -144,6 +144,8 @@ protected void handleLocalMessage(Message msg) { handle((GetVolumeBackingChainFromPrimaryStorageMsg) msg); } else if (msg instanceof DeleteVolumeChainOnPrimaryStorageMsg) { handle((DeleteVolumeChainOnPrimaryStorageMsg) msg); + } else if (msg instanceof EncryptVolumeBitsOnPrimaryStorageMsg) { + handle((EncryptVolumeBitsOnPrimaryStorageMsg) msg); } else { super.handleLocalMessage(msg); } @@ -255,6 +257,7 @@ protected void handle(InstantiateVolumeOnPrimaryStorageMsg msg) { spec.setUuid(volume.getUuid()); spec.setSize(volume.getSize()); spec.setAllocatedUrl(msg.getAllocatedInstallUrl()); + spec.setEncrypted(Boolean.TRUE.equals(volume.getEncrypted())); if (msg instanceof InstantiateTemporaryRootVolumeFromTemplateOnPrimaryStorageMsg) { // FIXME: use more accurate name spec.setName(buildReimageVolumeName(((InstantiateTemporaryRootVolumeFromTemplateOnPrimaryStorageMsg) msg).getOriginVolumeUuid())); @@ -281,8 +284,14 @@ protected void handle(CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg msg) { spec.setUuid(msg.getVolumeUuid()); spec.setName(buildVolumeName(msg.getVolumeUuid())); - Long volumeSize = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, msg.getVolumeUuid()).select(VolumeVO_.size).findValue(); - spec.setSize(volumeSize != null && volumeSize != 0 ? volumeSize : msg.getSnapshot().getSize()); + VolumeVO volume = dbf.findByUuid(msg.getVolumeUuid(), VolumeVO.class); + if (volume == null) { + reply.setError(operr("cannot find target volume[uuid:%s] to pass encryption attribute", msg.getVolumeUuid())); + bus.reply(msg, reply); + return; + } + spec.setSize(volume.getSize() != 0 ? volume.getSize() : msg.getSnapshot().getSize()); + spec.setEncrypted(volume.isEncrypted()); ReturnValueCompletion completion = new ReturnValueCompletion(msg) { @Override public void success(VolumeStats stats) { @@ -290,7 +299,8 @@ public void success(VolumeStats stats) { reply.setSize(stats.getSize()); reply.setInstallPath(stats.getInstallPath()); reply.setProtocol(externalVO.getDefaultProtocol()); - reply.setIncremental(!controller.reportCapabilities().getSnapshotCapability().isSupportLazyDelete()); + reply.setIncremental(stats.getParentUri() != null + && !controller.reportCapabilities().getSnapshotCapability().isSupportLazyDelete()); bus.reply(msg, reply); } @@ -434,6 +444,22 @@ public void fail(ErrorCode errorCode) { }); } + private void handle(EncryptVolumeBitsOnPrimaryStorageMsg msg) { + EncryptVolumeBitsOnPrimaryStorageReply reply = new EncryptVolumeBitsOnPrimaryStorageReply(); + controller.encryptVolumeBits(msg, new ReturnValueCompletion(msg) { + @Override + public void success(EncryptVolumeBitsOnPrimaryStorageReply returnValue) { + bus.reply(msg, returnValue); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + protected void handle(final SetTrashExpirationTimeMsg msg) { controller.setTrashExpireTime(msg.getExpirationTime(), new Completion(msg) { @Override @@ -534,7 +560,7 @@ public void fail(ErrorCode errorCode) { } protected void handle(ResizeVolumeOnPrimaryStorageMsg msg) { - controller.expandVolume(msg.getVolume().getInstallPath(), msg.getSize(), new ReturnValueCompletion(msg) { + controller.expandVolume(msg.getVolume(), msg.getSize(), new ReturnValueCompletion(msg) { final ResizeVolumeOnPrimaryStorageReply reply = new ResizeVolumeOnPrimaryStorageReply(); @Override @@ -562,7 +588,13 @@ protected void handle(TakeSnapshotMsg msg) { return; } - VolumeInventory vol = VolumeInventory.valueOf(dbf.findByUuid(sp.getVolumeUuid(), VolumeVO.class)); + VolumeVO volumeVO = dbf.findByUuid(sp.getVolumeUuid(), VolumeVO.class); + ZbsVolumeEncryptionExtensionPoint zbsEncryptionExtension = getZbsVolumeEncryptionExtension(); + if (zbsEncryptionExtension != null) { + zbsEncryptionExtension.beforeTakeSnapshot(self.getUuid(), volumeVO, sp); + } + + VolumeInventory vol = VolumeInventory.valueOf(volumeVO); CreateVolumeSnapshotSpec sspec = new CreateVolumeSnapshotSpec(); sspec.setVolumeInstallPath(vol.getInstallPath()); sspec.setName(buildSnapshotName(sp.getUuid())); @@ -573,6 +605,9 @@ public void success(VolumeSnapshotStats stats) { sp.setPrimaryStorageUuid(self.getUuid()); sp.setType(VolumeSnapshotConstant.STORAGE_SNAPSHOT_TYPE.toString()); sp.setSize(stats.getActualSize()); + if (zbsEncryptionExtension != null) { + zbsEncryptionExtension.afterTakeSnapshot(self.getUuid(), volumeVO, sp); + } reply.setInventory(sp); bus.reply(msg, reply); } @@ -585,6 +620,18 @@ public void fail(ErrorCode errorCode) { }); } + private ZbsVolumeEncryptionExtensionPoint getZbsVolumeEncryptionExtension() { + if (!ZbsVolumeEncryptionExtensionPoint.PRIMARY_STORAGE_IDENTITY.equals(controller.getIdentity())) { + return null; + } + + List exts = pluginRgty.getExtensionList(ZbsVolumeEncryptionExtensionPoint.class); + if (exts.isEmpty()) { + throw new OperationFailureException(operr("cannot find ZBS volume encryption extension")); + } + return exts.get(0); + } + private BlockExternalPrimaryStorageBackend getBlockBackend(String vendor) { BlockExternalPrimaryStorageFactory blockFactory = factory.blockExternalPrimaryStorageFactories.get(vendor); return blockFactory.getBlockExternalPrimaryStorageBackend(externalVO); @@ -745,7 +792,9 @@ public void handle(ErrorCode errCode, Map data) { protected void handle(CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg msg) { CreateImageCacheFromVolumeSnapshotOnPrimaryStorageReply reply = new CreateImageCacheFromVolumeSnapshotOnPrimaryStorageReply(); - boolean useSnapshotAsCache = msg.hasSystemTag(VolumeSystemTags.FAST_CREATE.getTagFormat()); + boolean targetEncrypted = Boolean.TRUE.equals(msg.getEncrypted()) || Boolean.TRUE.equals(msg.getVolumeSnapshot().getEncrypted()); + boolean useSnapshotAsCache = msg.hasSystemTag(VolumeSystemTags.FAST_CREATE.getTagFormat()) + && (!targetEncrypted || Boolean.TRUE.equals(msg.getVolumeSnapshot().getEncrypted())); if (useSnapshotAsCache) { ImageCacheVO cache = createTemporaryImageCacheFromVolumeSnapshot(msg.getImageInventory(), msg.getVolumeSnapshot()); dbf.persist(cache); @@ -789,6 +838,7 @@ public void run(FlowTrigger trigger, Map data) { spec.setName(buildImageName(image.getUuid())); spec.setUuid(msg.getImageInventory().getUuid()); spec.setSize(msg.getImageInventory().getSize()); + spec.setEncrypted(Boolean.TRUE.equals(msg.getEncrypted()) || Boolean.TRUE.equals(msg.getVolumeSnapshot().getEncrypted())); controller.copyVolume(msg.getVolumeSnapshot().getPrimaryStorageInstallPath(), spec, new ReturnValueCompletion(trigger) { @Override public void success(VolumeStats dst) { @@ -1896,6 +1946,7 @@ public void run(final FlowTrigger trigger, Map data) { spec.setName(buildReimageVolumeName(msg.getVolume().getUuid())); spec.setUuid(msg.getVolume().getUuid()); spec.setSize(msg.getVolume().getSize()); + spec.setEncrypted(Boolean.TRUE.equals(msg.getVolume().getEncrypted())); controller.cloneVolume(installUrl, spec, new ReturnValueCompletion(trigger) { @Override public void success(VolumeStats returnValue) { diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedEmptyVolumeCreator.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedEmptyVolumeCreator.java new file mode 100644 index 00000000000..d234c4fbf05 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedEmptyVolumeCreator.java @@ -0,0 +1,98 @@ +package org.zstack.storage.encrypt; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.storage.addon.primary.CreateVolumeSpec; +import org.zstack.header.storage.addon.primary.ZbsVolumeEncryptionBackend; +import org.zstack.header.volume.VolumeConstant; +import org.zstack.header.volume.VolumeStats; + +import static org.zstack.core.Platform.operr; + +public class ZbsEncryptedEmptyVolumeCreator { + @Autowired + private ZbsVolumeEncryptionMaterialFactory materialFactory; + @Autowired + private ZbsVolumeEncryptionKvmCaller kvmCaller; + @Autowired + private ZbsVolumeEncryptionCleanup cleanup; + + void create(ZbsVolumeEncryptionBackend backend, CreateVolumeSpec spec, + ReturnValueCompletion completion) { + backend.createLuksBackingVolume(backend.buildConfiguredVolumePath(spec.getName()), spec.getSize(), + new ReturnValueCompletion(completion) { + @Override + public void success(String installPath) { + createOnPreparedBackingWithCleanup(backend, installPath, spec, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void createOnPreparedBackingWithCleanup(ZbsVolumeEncryptionBackend backend, String installPath, + CreateVolumeSpec spec, + ReturnValueCompletion completion) { + createOnPreparedBacking(backend.getPrimaryStorageUuid(), installPath, spec, + new ReturnValueCompletion(completion) { + @Override + public void success(VolumeStats returnValue) { + completion.success(returnValue); + } + + @Override + public void fail(ErrorCode errorCode) { + backend.deleteLuksBackingVolume(installPath); + completion.fail(errorCode); + } + }); + } + + private void createOnPreparedBacking(String primaryStorageUuid, String installPath, CreateVolumeSpec spec, + ReturnValueCompletion completion) { + ZbsVolumeEncryptionMaterial material; + try { + material = materialFactory.prepareVolumeEncryption(primaryStorageUuid, spec.getUuid(), + String.format("create encrypted volume[uuid:%s]", spec.getUuid())); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + return; + } catch (Exception e) { + completion.fail(operr("failed to prepare encrypted ZBS volume[uuid:%s] create: %s", + spec.getUuid(), e.getMessage())); + return; + } + + ZbsVolumeEncryptionCommands.KVMHostLuksCreateEmptyCmd cmd = + new ZbsVolumeEncryptionCommands.KVMHostLuksCreateEmptyCmd(); + cmd.psUuid = primaryStorageUuid; + cmd.installPath = installPath; + cmd.size = spec.getSize(); + cmd.encryptedDek = material.encryptedDek; + + kvmCaller.call(material.hostUuid, ZbsVolumeEncryptionConstants.KVM_HOST_LUKS_CREATE_EMPTY_PATH, + cmd, ZbsVolumeEncryptionCommands.KVMHostLuksRsp.class, "luks createempty", + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsVolumeEncryptionCommands.KVMHostLuksRsp rsp) { + VolumeStats stats = new VolumeStats(); + stats.setInstallPath(cmd.installPath); + stats.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); + stats.setSize(spec.getSize()); + stats.setActualSize(rsp.actualSize == null ? 0 : rsp.actualSize); + completion.success(stats); + } + + @Override + public void fail(ErrorCode errorCode) { + cleanup.cleanupVolume(primaryStorageUuid, cmd.installPath); + completion.fail(errorCode); + } + }); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedSnapshotVolumeCopier.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedSnapshotVolumeCopier.java new file mode 100644 index 00000000000..65e21e80955 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedSnapshotVolumeCopier.java @@ -0,0 +1,135 @@ +package org.zstack.storage.encrypt; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.db.Q; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.storage.addon.primary.CreateVolumeSpec; +import org.zstack.header.storage.addon.primary.ZbsVolumeEncryptionBackend; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO_; +import org.zstack.header.volume.VolumeStats; +import org.zstack.header.volume.VolumeVO; +import org.zstack.header.volume.VolumeVO_; + +import static org.zstack.core.Platform.operr; + +public class ZbsEncryptedSnapshotVolumeCopier { + @Autowired + private ZbsVolumeEncryptionMaterialFactory materialFactory; + @Autowired + private ZbsEncryptedVolumeCloner cloner; + + void copy(ZbsVolumeEncryptionBackend backend, String srcInstallPath, CreateVolumeSpec dst, + ReturnValueCompletion completion) { + backend.resolveSnapshotPathForQemu(srcInstallPath, new ReturnValueCompletion(completion) { + @Override + public void success(String resolvedSrcInstallPath) { + createBackingAndCopy(backend, srcInstallPath, resolvedSrcInstallPath, dst, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void createBackingAndCopy(ZbsVolumeEncryptionBackend backend, String srcInstallPath, + String resolvedSrcInstallPath, CreateVolumeSpec dst, + ReturnValueCompletion completion) { + long virtualSize; + try { + virtualSize = resolveSnapshotCopyVirtualSize(backend, srcInstallPath, dst); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + return; + } + + if (dst.getSize() <= 0) { + dst.setSize(virtualSize); + } + + backend.createLuksBackingVolume(backend.buildConfiguredVolumePath(dst.getName()), virtualSize, + new ReturnValueCompletion(completion) { + @Override + public void success(String dstInstallPath) { + copyWithCleanup(backend, srcInstallPath, resolvedSrcInstallPath, dstInstallPath, dst, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private long resolveSnapshotCopyVirtualSize(ZbsVolumeEncryptionBackend backend, String srcInstallPath, + CreateVolumeSpec dst) { + if (dst.getSize() > 0) { + return dst.getSize(); + } + + VolumeSnapshotVO snapshot = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.primaryStorageUuid, backend.getPrimaryStorageUuid()) + .eq(VolumeSnapshotVO_.primaryStorageInstallPath, srcInstallPath) + .find(); + if (snapshot == null) { + throw new OperationFailureException(operr( + "cannot find source snapshot by installPath[%s] on external primary storage[uuid:%s] to prepare encrypted volume[uuid:%s]", + srcInstallPath, backend.getPrimaryStorageUuid(), dst.getUuid())); + } + + Long volumeSize = Q.New(VolumeVO.class) + .select(VolumeVO_.size) + .eq(VolumeVO_.uuid, snapshot.getVolumeUuid()) + .findValue(); + if (volumeSize != null && volumeSize > 0) { + return volumeSize; + } + Long snapshotSize = snapshot.getSize(); + if (snapshotSize != null && snapshotSize > 0) { + return snapshotSize; + } + + throw new OperationFailureException(operr( + "cannot determine virtual size for encrypted ZBS volume[uuid:%s] from snapshot[uuid:%s, installPath:%s]", + dst.getUuid(), snapshot.getUuid(), srcInstallPath)); + } + + private void copyWithCleanup(ZbsVolumeEncryptionBackend backend, String srcInstallPath, String resolvedSrcInstallPath, + String dstInstallPath, CreateVolumeSpec dst, + ReturnValueCompletion completion) { + copy(backend.getPrimaryStorageUuid(), srcInstallPath, resolvedSrcInstallPath, dstInstallPath, dst, + new ReturnValueCompletion(completion) { + @Override + public void success(VolumeStats returnValue) { + completion.success(returnValue); + } + + @Override + public void fail(ErrorCode errorCode) { + backend.deleteLuksBackingVolume(dstInstallPath); + completion.fail(errorCode); + } + }); + } + + private void copy(String primaryStorageUuid, String srcInstallPath, String resolvedSrcInstallPath, + String dstInstallPath, CreateVolumeSpec dst, ReturnValueCompletion completion) { + ZbsVolumeEncryptionMaterial material; + try { + material = materialFactory.prepareSnapshotCopyEncryption(primaryStorageUuid, srcInstallPath, dst); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + return; + } catch (Exception e) { + completion.fail(operr("failed to prepare encrypted ZBS copy from snapshot[%s] to target[uuid:%s]: %s", + srcInstallPath, dst.getUuid(), e.getMessage())); + return; + } + + cloner.clone(primaryStorageUuid, resolvedSrcInstallPath, dstInstallPath, dst, material, completion); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedVolumeCloner.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedVolumeCloner.java new file mode 100644 index 00000000000..44e69e2fc1f --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedVolumeCloner.java @@ -0,0 +1,305 @@ +package org.zstack.storage.encrypt; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.db.Q; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.storage.addon.primary.CreateVolumeSpec; +import org.zstack.header.storage.addon.primary.ZbsVolumeEncryptionBackend; +import org.zstack.header.storage.primary.ImageCacheVO; +import org.zstack.header.storage.primary.ImageCacheVO_; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO_; +import org.zstack.header.volume.VolumeConstant; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeStats; +import org.zstack.header.volume.VolumeVO; +import org.zstack.header.volume.VolumeVO_; + +import static org.zstack.core.Platform.operr; + +public class ZbsEncryptedVolumeCloner { + @Autowired + private ZbsVolumeEncryptionMaterialFactory materialFactory; + @Autowired + private ZbsVolumeEncryptionKvmCaller kvmCaller; + @Autowired + private ZbsVolumeEncryptionCleanup cleanup; + @Autowired + private ZbsEncryptedVolumeResizer volumeResizer; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; + + void cloneFromImage(ZbsVolumeEncryptionBackend backend, String srcInstallPath, CreateVolumeSpec dst, + ReturnValueCompletion completion) { + if (isEncryptedSource(backend, srcInstallPath)) { + cloneEncryptedSourceAsBacking(backend, srcInstallPath, dst, completion); + return; + } + + backend.resolveSnapshotPathForQemu(srcInstallPath, new ReturnValueCompletion(completion) { + @Override + public void success(String resolvedSrcInstallPath) { + createBackingAndClone(backend, srcInstallPath, resolvedSrcInstallPath, dst, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void createBackingAndClone(ZbsVolumeEncryptionBackend backend, String srcInstallPath, + String resolvedSrcInstallPath, CreateVolumeSpec dst, + ReturnValueCompletion completion) { + resolveCloneVirtualSize(backend, srcInstallPath, resolvedSrcInstallPath, dst, + new ReturnValueCompletion(completion) { + @Override + public void success(Long virtualSize) { + createBackingAndClone(backend, resolvedSrcInstallPath, dst, virtualSize, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void createBackingAndClone(ZbsVolumeEncryptionBackend backend, String srcInstallPath, CreateVolumeSpec dst, + long virtualSize, ReturnValueCompletion completion) { + backend.createLuksBackingVolume(backend.buildConfiguredVolumePath(dst.getName()), virtualSize, + new ReturnValueCompletion(completion) { + @Override + public void success(String dstInstallPath) { + cloneFromImageWithCleanup(backend, srcInstallPath, dstInstallPath, dst, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void cloneEncryptedSourceAsBacking(ZbsVolumeEncryptionBackend backend, String srcInstallPath, + CreateVolumeSpec dst, + ReturnValueCompletion completion) { + resolveCloneVirtualSize(backend, srcInstallPath, srcInstallPath, dst, + new ReturnValueCompletion(completion) { + @Override + public void success(Long virtualSize) { + dst.setSize(virtualSize); + backend.cloneVolumeAsBacking(srcInstallPath, dst, + new ReturnValueCompletion(completion) { + @Override + public void success(VolumeStats stats) { + resizeClonedEncryptedBacking(backend, stats, dst, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void resizeClonedEncryptedBacking(ZbsVolumeEncryptionBackend backend, VolumeStats stats, + CreateVolumeSpec dst, + ReturnValueCompletion completion) { + VolumeInventory volume = new VolumeInventory(); + volume.setUuid(dst.getUuid()); + volume.setInstallPath(stats.getInstallPath()); + volume.setSize(0); + volume.setActualSize(stats.getActualSize()); + volume.setEncrypted(true); + + volumeResizer.resize(backend.getPrimaryStorageUuid(), volume, dst.getSize(), + new ReturnValueCompletion(completion) { + @Override + public void success(VolumeStats returnValue) { + returnValue.setParentUri(stats.getParentUri()); + completion.success(returnValue); + } + + @Override + public void fail(ErrorCode errorCode) { + backend.deleteLuksBackingVolume(stats.getInstallPath()); + completion.fail(errorCode); + } + }); + } + + private void resolveCloneVirtualSize(ZbsVolumeEncryptionBackend backend, String srcInstallPath, + String resolvedSrcInstallPath, CreateVolumeSpec dst, + ReturnValueCompletion completion) { + if (dst.getSize() > 0) { + completion.success(dst.getSize()); + return; + } + + Long dbSize = findSourceVirtualSizeInDb(backend, srcInstallPath); + if (dbSize != null && dbSize > 0) { + dst.setSize(dbSize); + completion.success(dbSize); + return; + } + + backend.stats(resolvedSrcInstallPath, new ReturnValueCompletion(completion) { + @Override + public void success(VolumeStats stats) { + Long size = stats.getSize(); + if (size == null || size <= 0) { + completion.fail(operr( + "cannot determine virtual size for encrypted ZBS volume[uuid:%s] from source[%s]", + dst.getUuid(), srcInstallPath)); + return; + } + + dst.setSize(size); + completion.success(size); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private Long findSourceVirtualSizeInDb(ZbsVolumeEncryptionBackend backend, String srcInstallPath) { + VolumeSnapshotVO snapshot = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.primaryStorageUuid, backend.getPrimaryStorageUuid()) + .eq(VolumeSnapshotVO_.primaryStorageInstallPath, srcInstallPath) + .find(); + if (snapshot != null) { + Long volumeSize = Q.New(VolumeVO.class) + .select(VolumeVO_.size) + .eq(VolumeVO_.uuid, snapshot.getVolumeUuid()) + .findValue(); + if (volumeSize != null && volumeSize > 0) { + return volumeSize; + } + return snapshot.getSize() > 0 ? snapshot.getSize() : null; + } + + Long volumeSize = Q.New(VolumeVO.class) + .select(VolumeVO_.size) + .eq(VolumeVO_.primaryStorageUuid, backend.getPrimaryStorageUuid()) + .eq(VolumeVO_.installPath, srcInstallPath) + .findValue(); + if (volumeSize != null && volumeSize > 0) { + return volumeSize; + } + + return Q.New(ImageCacheVO.class) + .select(ImageCacheVO_.size) + .eq(ImageCacheVO_.primaryStorageUuid, backend.getPrimaryStorageUuid()) + .eq(ImageCacheVO_.installUrl, srcInstallPath) + .findValue(); + } + + private boolean isEncryptedSource(ZbsVolumeEncryptionBackend backend, String srcInstallPath) { + Boolean snapshotEncrypted = Q.New(VolumeSnapshotVO.class) + .select(VolumeSnapshotVO_.encrypted) + .eq(VolumeSnapshotVO_.primaryStorageUuid, backend.getPrimaryStorageUuid()) + .eq(VolumeSnapshotVO_.primaryStorageInstallPath, srcInstallPath) + .findValue(); + if (Boolean.TRUE.equals(snapshotEncrypted)) { + return true; + } + + Boolean volumeEncrypted = Q.New(VolumeVO.class) + .select(VolumeVO_.encrypted) + .eq(VolumeVO_.primaryStorageUuid, backend.getPrimaryStorageUuid()) + .eq(VolumeVO_.installPath, srcInstallPath) + .findValue(); + if (Boolean.TRUE.equals(volumeEncrypted)) { + return true; + } + + String imageUuid = Q.New(ImageCacheVO.class) + .select(ImageCacheVO_.imageUuid) + .eq(ImageCacheVO_.primaryStorageUuid, backend.getPrimaryStorageUuid()) + .eq(ImageCacheVO_.installUrl, srcInstallPath) + .findValue(); + return imageUuid != null && snapshotEncryptionHelper.hasTemporarySnapshotImageKey(imageUuid); + } + + private void cloneFromImageWithCleanup(ZbsVolumeEncryptionBackend backend, String srcInstallPath, + String dstInstallPath, CreateVolumeSpec dst, + ReturnValueCompletion completion) { + cloneFromImage(backend.getPrimaryStorageUuid(), srcInstallPath, dstInstallPath, dst, + new ReturnValueCompletion(completion) { + @Override + public void success(VolumeStats returnValue) { + completion.success(returnValue); + } + + @Override + public void fail(ErrorCode errorCode) { + backend.deleteLuksBackingVolume(dstInstallPath); + completion.fail(errorCode); + } + }); + } + + void cloneFromImage(String primaryStorageUuid, String srcInstallPath, String dstInstallPath, + CreateVolumeSpec dst, ReturnValueCompletion completion) { + ZbsVolumeEncryptionMaterial material; + try { + material = materialFactory.prepareVolumeEncryption(primaryStorageUuid, dst.getUuid(), + String.format("clone encrypted volume[uuid:%s] from image", dst.getUuid())); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + return; + } catch (Exception e) { + completion.fail(operr("failed to prepare encrypted ZBS volume[uuid:%s] clone from image: %s", + dst.getUuid(), e.getMessage())); + return; + } + + clone(primaryStorageUuid, srcInstallPath, dstInstallPath, dst, material, completion); + } + + void clone(String primaryStorageUuid, String srcInstallPath, String dstInstallPath, + CreateVolumeSpec dst, ZbsVolumeEncryptionMaterial material, + ReturnValueCompletion completion) { + ZbsVolumeEncryptionCommands.KVMHostLuksCloneCmd cmd = + new ZbsVolumeEncryptionCommands.KVMHostLuksCloneCmd(); + cmd.psUuid = primaryStorageUuid; + cmd.srcPath = srcInstallPath; + cmd.dstPath = dstInstallPath; + cmd.encryptedDek = material.encryptedDek; + cmd.virtualSizeForLuksClone = dst.getSize(); + + kvmCaller.call(material.hostUuid, ZbsVolumeEncryptionConstants.KVM_HOST_LUKS_CLONE_PATH, + cmd, ZbsVolumeEncryptionCommands.KVMHostLuksRsp.class, "luks clone", + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsVolumeEncryptionCommands.KVMHostLuksRsp rsp) { + VolumeStats stats = new VolumeStats(); + stats.setInstallPath(cmd.dstPath); + stats.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); + stats.setSize(dst.getSize()); + stats.setActualSize(rsp.actualSize == null ? 0 : rsp.actualSize); + completion.success(stats); + } + + @Override + public void fail(ErrorCode errorCode) { + cleanup.cleanupVolume(primaryStorageUuid, cmd.dstPath); + completion.fail(errorCode); + } + }); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedVolumeResizer.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedVolumeResizer.java new file mode 100644 index 00000000000..fdac65d9430 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedVolumeResizer.java @@ -0,0 +1,96 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.volume.VolumeConstant; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeStats; + +import static org.zstack.core.Platform.operr; + +public class ZbsEncryptedVolumeResizer { + @Autowired + private ZbsVolumeEncryptionMaterialFactory materialFactory; + @Autowired + private ZbsVolumeEncryptionKvmCaller kvmCaller; + + void resize(String primaryStorageUuid, VolumeInventory volume, long size, + ReturnValueCompletion completion) { + if (StringUtils.isBlank(primaryStorageUuid)) { + completion.fail(operr("prepare encrypted ZBS volume resize requires primary storage uuid")); + return; + } + if (volume == null || StringUtils.isBlank(volume.getUuid())) { + completion.fail(operr("prepare encrypted ZBS volume resize requires target volume inventory")); + return; + } + if (!Boolean.TRUE.equals(volume.getEncrypted())) { + completion.fail(operr("ZBS volume[uuid:%s, installPath:%s] is not encrypted", + volume.getUuid(), volume.getInstallPath())); + return; + } + + long currentSize = volume.getSize(); + if (currentSize > 0 && size < currentSize) { + completion.fail(operr("cannot shrink ZBS volume[uuid:%s, installPath:%s] from %s to %s", + volume.getUuid(), volume.getInstallPath(), currentSize, size)); + return; + } + if (currentSize > 0 && size == currentSize) { + VolumeStats stats = new VolumeStats(); + stats.setInstallPath(volume.getInstallPath()); + stats.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); + stats.setSize(size); + stats.setActualSize(volume.getActualSize()); + completion.success(stats); + return; + } + + ZbsVolumeEncryptionMaterial material; + try { + material = materialFactory.prepareVolumeEncryption(primaryStorageUuid, volume.getUuid(), + String.format("resize encrypted volume[uuid:%s]", volume.getUuid())); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + return; + } catch (Exception e) { + completion.fail(operr("failed to prepare encrypted ZBS volume[uuid:%s, installPath:%s] resize: %s", + volume.getUuid(), volume.getInstallPath(), e.getMessage())); + return; + } + if (StringUtils.isBlank(material.hostUuid) || StringUtils.isBlank(material.encryptedDek)) { + completion.fail(operr("cannot resize encrypted ZBS volume[uuid:%s, installPath:%s], encryption resize info is missing", + volume.getUuid(), volume.getInstallPath())); + return; + } + + ZbsVolumeEncryptionCommands.KVMHostLuksResizeCmd cmd = + new ZbsVolumeEncryptionCommands.KVMHostLuksResizeCmd(); + cmd.psUuid = primaryStorageUuid; + cmd.installPath = volume.getInstallPath(); + cmd.encryptedDek = material.encryptedDek; + cmd.virtualSize = size; + + kvmCaller.call(material.hostUuid, ZbsVolumeEncryptionConstants.KVM_HOST_LUKS_RESIZE_PATH, + cmd, ZbsVolumeEncryptionCommands.KVMHostLuksRsp.class, "luks resize", + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsVolumeEncryptionCommands.KVMHostLuksRsp rsp) { + VolumeStats stats = new VolumeStats(); + stats.setInstallPath(volume.getInstallPath()); + stats.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); + stats.setSize(size); + stats.setActualSize(rsp.actualSize == null ? 0 : rsp.actualSize); + completion.success(stats); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptInPlaceHandler.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptInPlaceHandler.java new file mode 100644 index 00000000000..20216a3dca5 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptInPlaceHandler.java @@ -0,0 +1,141 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.storage.addon.primary.ZbsVolumeEncryptionBackend; +import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageReply; +import org.zstack.header.volume.VolumeStats; + +import static org.zstack.core.Platform.operr; + +public class ZbsVolumeEncryptInPlaceHandler { + @Autowired + private ZbsVolumeEncryptionKvmCaller kvmCaller; + + void encrypt(ZbsVolumeEncryptionBackend backend, EncryptVolumeBitsOnPrimaryStorageMsg msg, + ReturnValueCompletion completion) { + String targetInstallPath; + try { + targetInstallPath = backend.buildEncryptedTargetPath(msg.getInstallPath()); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + return; + } + + checkNoSnapshots(backend, msg, targetInstallPath, completion); + } + + private void checkNoSnapshots(ZbsVolumeEncryptionBackend backend, EncryptVolumeBitsOnPrimaryStorageMsg msg, + String targetInstallPath, + ReturnValueCompletion completion) { + backend.checkNoSnapshots(msg.getInstallPath(), new Completion(completion) { + @Override + public void success() { + querySourceStats(backend, msg, targetInstallPath, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void querySourceStats(ZbsVolumeEncryptionBackend backend, EncryptVolumeBitsOnPrimaryStorageMsg msg, + String targetInstallPath, + ReturnValueCompletion completion) { + backend.stats(msg.getInstallPath(), new ReturnValueCompletion(completion) { + @Override + public void success(VolumeStats sourceStats) { + createTargetBacking(backend, msg, targetInstallPath, sourceStats, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void createTargetBacking(ZbsVolumeEncryptionBackend backend, EncryptVolumeBitsOnPrimaryStorageMsg msg, + String targetInstallPath, VolumeStats sourceStats, + ReturnValueCompletion completion) { + backend.createLuksBackingVolume(targetInstallPath, sourceStats.getSize(), new ReturnValueCompletion(completion) { + @Override + public void success(String createdTargetInstallPath) { + encryptWithCleanup(backend, msg, createdTargetInstallPath, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void encryptWithCleanup(ZbsVolumeEncryptionBackend backend, EncryptVolumeBitsOnPrimaryStorageMsg msg, + String targetInstallPath, + ReturnValueCompletion completion) { + msg.setTargetInstallPath(targetInstallPath); + encrypt(backend.getPrimaryStorageUuid(), msg, + new ReturnValueCompletion(completion) { + @Override + public void success(EncryptVolumeBitsOnPrimaryStorageReply returnValue) { + completion.success(returnValue); + } + + @Override + public void fail(ErrorCode errorCode) { + backend.deleteLuksBackingVolume(targetInstallPath); + completion.fail(errorCode); + } + }); + } + + private void encrypt(String primaryStorageUuid, EncryptVolumeBitsOnPrimaryStorageMsg msg, + ReturnValueCompletion completion) { + if (StringUtils.isBlank(msg.getHostUuid())) { + completion.fail(operr("zbs encryptInPlace requires hostUuid; volume[uuid:%s] installPath[%s] has none", + msg.getVolumeUuid(), msg.getInstallPath())); + return; + } + if (StringUtils.isBlank(msg.getEncryptedDek())) { + completion.fail(operr("zbs encryptInPlace requires encryptedDek; volume[uuid:%s] installPath[%s] has none", + msg.getVolumeUuid(), msg.getInstallPath())); + return; + } + if (StringUtils.isBlank(msg.getTargetInstallPath())) { + completion.fail(operr("zbs encryptInPlace requires targetInstallPath; volume[uuid:%s] installPath[%s] has none", + msg.getVolumeUuid(), msg.getInstallPath())); + return; + } + + ZbsVolumeEncryptionCommands.KVMHostEncryptInPlaceCmd cmd = + new ZbsVolumeEncryptionCommands.KVMHostEncryptInPlaceCmd(); + cmd.psUuid = primaryStorageUuid; + cmd.installPath = msg.getInstallPath(); + cmd.targetInstallPath = msg.getTargetInstallPath(); + cmd.encryptedDek = msg.getEncryptedDek(); + + kvmCaller.call(msg.getHostUuid(), ZbsVolumeEncryptionConstants.KVM_HOST_LUKS_ENCRYPT_IN_PLACE_PATH, + cmd, ZbsVolumeEncryptionCommands.KVMHostLuksRsp.class, "encryptInPlace", + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsVolumeEncryptionCommands.KVMHostLuksRsp rsp) { + EncryptVolumeBitsOnPrimaryStorageReply er = new EncryptVolumeBitsOnPrimaryStorageReply(); + er.setInstallPath(StringUtils.defaultIfBlank(rsp.installPath, msg.getInstallPath())); + completion.success(er); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionCleanup.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionCleanup.java new file mode 100644 index 00000000000..5ffeedc25de --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionCleanup.java @@ -0,0 +1,32 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.header.message.MessageReply; +import org.zstack.header.storage.primary.DeleteBitsOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.PrimaryStorageConstant; + +public class ZbsVolumeEncryptionCleanup { + @Autowired + private CloudBus bus; + + @SuppressWarnings("deprecation") + void cleanupVolume(String primaryStorageUuid, String installPath) { + if (StringUtils.isBlank(primaryStorageUuid) || StringUtils.isBlank(installPath)) { + return; + } + + DeleteBitsOnPrimaryStorageMsg msg = new DeleteBitsOnPrimaryStorageMsg(); + msg.setPrimaryStorageUuid(primaryStorageUuid); + msg.setInstallPath(installPath); + msg.setRecycle(false); + bus.makeTargetServiceIdByResourceUuid(msg, PrimaryStorageConstant.SERVICE_ID, primaryStorageUuid); + bus.send(msg, new CloudBusCallBack(null) { + @Override + public void run(MessageReply reply) { + } + }); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionCommands.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionCommands.java new file mode 100644 index 00000000000..2626488a033 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionCommands.java @@ -0,0 +1,47 @@ +package org.zstack.storage.encrypt; + +import org.zstack.header.log.NoLogging; +import org.zstack.kvm.KVMAgentCommands; + +final class ZbsVolumeEncryptionCommands { + private ZbsVolumeEncryptionCommands() { + } + + static class KVMHostLuksCloneCmd extends KVMAgentCommands.AgentCommand { + public String psUuid; + @NoLogging + public String encryptedDek; + public String srcPath; + public String dstPath; + public Long virtualSizeForLuksClone; + } + + static class KVMHostLuksCreateEmptyCmd extends KVMAgentCommands.AgentCommand { + public String psUuid; + @NoLogging + public String encryptedDek; + public String installPath; + public long size; + } + + static class KVMHostEncryptInPlaceCmd extends KVMAgentCommands.AgentCommand { + public String psUuid; + @NoLogging + public String encryptedDek; + public String installPath; + public String targetInstallPath; + } + + static class KVMHostLuksResizeCmd extends KVMAgentCommands.AgentCommand { + public String psUuid; + @NoLogging + public String encryptedDek; + public String installPath; + public Long virtualSize; + } + + static class KVMHostLuksRsp extends KVMAgentCommands.AgentResponse { + public Long actualSize; + public String installPath; + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionConstants.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionConstants.java new file mode 100644 index 00000000000..1fdcb3fd8f6 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionConstants.java @@ -0,0 +1,11 @@ +package org.zstack.storage.encrypt; + +final class ZbsVolumeEncryptionConstants { + static final String KVM_HOST_LUKS_CLONE_PATH = "/zbs/primarystorage/kvmhost/luksclone"; + static final String KVM_HOST_LUKS_CREATE_EMPTY_PATH = "/zbs/primarystorage/kvmhost/lukscreateempty"; + static final String KVM_HOST_LUKS_ENCRYPT_IN_PLACE_PATH = "/zbs/primarystorage/kvmhost/encryptinplace"; + static final String KVM_HOST_LUKS_RESIZE_PATH = "/zbs/primarystorage/kvmhost/luksresize"; + + private ZbsVolumeEncryptionConstants() { + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionExtension.java new file mode 100644 index 00000000000..0cb4033479c --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionExtension.java @@ -0,0 +1,74 @@ +package org.zstack.storage.encrypt; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.storage.addon.primary.CreateVolumeSpec; +import org.zstack.header.storage.addon.primary.ZbsVolumeEncryptionBackend; +import org.zstack.header.storage.addon.primary.ZbsVolumeEncryptionExtensionPoint; +import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageReply; +import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeStats; +import org.zstack.header.volume.VolumeVO; + +public class ZbsVolumeEncryptionExtension implements ZbsVolumeEncryptionExtensionPoint { + @Autowired + private ZbsEncryptedEmptyVolumeCreator emptyVolumeCreator; + @Autowired + private ZbsEncryptedVolumeCloner volumeCloner; + @Autowired + private ZbsEncryptedSnapshotVolumeCopier snapshotVolumeCopier; + @Autowired + private ZbsVolumeEncryptInPlaceHandler encryptInPlaceHandler; + @Autowired + private ZbsEncryptedVolumeResizer volumeResizer; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; + + @Override + public void createEncryptedEmptyVolume(ZbsVolumeEncryptionBackend backend, CreateVolumeSpec spec, + ReturnValueCompletion completion) { + emptyVolumeCreator.create(backend, spec, completion); + } + + @Override + public void cloneEncryptedVolumeFromImage(ZbsVolumeEncryptionBackend backend, String srcInstallPath, + CreateVolumeSpec dst, ReturnValueCompletion completion) { + volumeCloner.cloneFromImage(backend, srcInstallPath, dst, completion); + } + + @Override + public void copyEncryptedVolumeFromSnapshot(ZbsVolumeEncryptionBackend backend, String srcInstallPath, + CreateVolumeSpec dst, ReturnValueCompletion completion) { + snapshotVolumeCopier.copy(backend, srcInstallPath, dst, completion); + } + + @Override + public void encryptVolumeBits(ZbsVolumeEncryptionBackend backend, EncryptVolumeBitsOnPrimaryStorageMsg msg, + ReturnValueCompletion completion) { + encryptInPlaceHandler.encrypt(backend, msg, completion); + } + + @Override + public void resizeEncryptedVolume(String primaryStorageUuid, VolumeInventory volume, long size, + ReturnValueCompletion completion) { + volumeResizer.resize(primaryStorageUuid, volume, size, completion); + } + + @Override + public void beforeTakeSnapshot(String primaryStorageUuid, VolumeVO volume, VolumeSnapshotInventory snapshot) { + if (volume == null || !volume.isEncrypted()) { + return; + } + snapshotEncryptionHelper.inheritVolumeKeyToSnapshot(volume, snapshot); + } + + @Override + public void afterTakeSnapshot(String primaryStorageUuid, VolumeVO volume, VolumeSnapshotInventory snapshot) { + if (volume == null || !volume.isEncrypted()) { + return; + } + snapshotEncryptionHelper.completeTakeSnapshot(volume, snapshot); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionKvmCaller.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionKvmCaller.java new file mode 100644 index 00000000000..51588903afc --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionKvmCaller.java @@ -0,0 +1,55 @@ +package org.zstack.storage.encrypt; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.host.HostConstant; +import org.zstack.header.message.MessageReply; +import org.zstack.kvm.KVMAgentCommands; +import org.zstack.kvm.KVMHostAsyncHttpCallMsg; +import org.zstack.kvm.KVMHostAsyncHttpCallReply; + +import static org.zstack.core.Platform.operr; + +public class ZbsVolumeEncryptionKvmCaller { + @Autowired + private CloudBus bus; + + void call(String hostUuid, String path, + KVMAgentCommands.AgentCommand cmd, + Class rspClass, + String operation, + ReturnValueCompletion completion) { + KVMHostAsyncHttpCallMsg kmsg = new KVMHostAsyncHttpCallMsg(); + kmsg.setCommand(cmd); + kmsg.setPath(path); + kmsg.setHostUuid(hostUuid); + kmsg.setNoStatusCheck(true); + bus.makeTargetServiceIdByResourceUuid(kmsg, HostConstant.SERVICE_ID, hostUuid); + bus.send(kmsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + + KVMHostAsyncHttpCallReply kr = reply.castReply(); + T rsp = kr.toResponse(rspClass); + if (rsp == null) { + completion.fail(operr("kvm host[uuid:%s] returned null reply for zbs %s path[%s]", + hostUuid, operation, path)); + return; + } + if (!rsp.isSuccess()) { + completion.fail(operr("kvm host[uuid:%s] zbs %s path[%s] failed, because: %s", + hostUuid, operation, path, rsp.getError())); + return; + } + + completion.success(rsp); + } + }); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionMaterial.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionMaterial.java new file mode 100644 index 00000000000..06578d216a7 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionMaterial.java @@ -0,0 +1,11 @@ +package org.zstack.storage.encrypt; + +class ZbsVolumeEncryptionMaterial { + final String hostUuid; + final String encryptedDek; + + ZbsVolumeEncryptionMaterial(String hostUuid, String encryptedDek) { + this.hostUuid = hostUuid; + this.encryptedDek = encryptedDek; + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionMaterialFactory.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionMaterialFactory.java new file mode 100644 index 00000000000..9ddcc9f5f06 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionMaterialFactory.java @@ -0,0 +1,83 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.db.Q; +import org.zstack.header.errorcode.ErrorableValue; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.storage.addon.primary.CreateVolumeSpec; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO_; +import org.zstack.header.volume.VolumeVO; +import org.zstack.header.volume.VolumeVO_; +import org.zstack.storage.volume.VolumeManager; + +import static org.zstack.core.Platform.operr; + +public class ZbsVolumeEncryptionMaterialFactory { + @Autowired + private VolumeEncryptedSecretHelper volumeEncryptedSecretHelper; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; + @Autowired + private VolumeManager volumeManager; + + ZbsVolumeEncryptionMaterial prepareVolumeEncryption(String primaryStorageUuid, String volumeUuid, String operation) { + if (StringUtils.isBlank(volumeUuid)) { + throw new OperationFailureException(operr("prepare encrypted ZBS volume requires target volume uuid")); + } + + String hostUuid = findConnectedKvmHost(primaryStorageUuid, operation); + String encryptedDek = volumeEncryptedSecretHelper.materializeAndSealVolumeDekForHost(hostUuid, volumeUuid); + if (StringUtils.isBlank(encryptedDek)) { + throw new OperationFailureException(operr( + "cannot prepare LUKS encryptedDek for encrypted volume[uuid:%s] on host[uuid:%s]", + volumeUuid, hostUuid)); + } + return new ZbsVolumeEncryptionMaterial(hostUuid, encryptedDek); + } + + ZbsVolumeEncryptionMaterial prepareSnapshotCopyEncryption(String primaryStorageUuid, String srcInstallPath, + CreateVolumeSpec dst) { + if (StringUtils.isBlank(dst.getUuid())) { + throw new OperationFailureException(operr("cannot copy encrypted ZBS volume from snapshot, target uuid is missing")); + } + + boolean targetVolume = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, dst.getUuid()).isExists(); + if (targetVolume) { + return prepareVolumeEncryption(primaryStorageUuid, dst.getUuid(), + String.format("copy encrypted volume[uuid:%s] from snapshot", dst.getUuid())); + } + + String snapshotUuid = Q.New(VolumeSnapshotVO.class) + .select(VolumeSnapshotVO_.uuid) + .eq(VolumeSnapshotVO_.primaryStorageUuid, primaryStorageUuid) + .eq(VolumeSnapshotVO_.primaryStorageInstallPath, srcInstallPath) + .findValue(); + if (StringUtils.isBlank(snapshotUuid)) { + throw new OperationFailureException(operr( + "cannot find source snapshot by installPath[%s] on external primary storage[uuid:%s] to prepare encrypted snapshot image cache[uuid:%s]", + srcInstallPath, primaryStorageUuid, dst.getUuid())); + } + + String hostUuid = findConnectedKvmHost(primaryStorageUuid, String.format( + "prepare LUKS secret for snapshot image cache[uuid:%s]", dst.getUuid())); + String encryptedDek = snapshotEncryptionHelper.prepareTemporarySnapshotImageEncryptedDek( + hostUuid, snapshotUuid, dst.getUuid(), true); + if (StringUtils.isBlank(encryptedDek)) { + throw new OperationFailureException(operr( + "cannot prepare LUKS encryptedDek for snapshot image cache[uuid:%s] from snapshot[uuid:%s] on host[uuid:%s]", + dst.getUuid(), snapshotUuid, hostUuid)); + } + return new ZbsVolumeEncryptionMaterial(hostUuid, encryptedDek); + } + + private String findConnectedKvmHost(String primaryStorageUuid, String operation) { + ErrorableValue host = volumeManager.findConnectedKvmHostByPrimaryStorage( + primaryStorageUuid, "external primary storage", operation); + if (!host.isSuccess()) { + throw new OperationFailureException(host.error); + } + return host.result; + } +} diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeInPlaceEncryptor.java b/storage/src/main/java/org/zstack/storage/volume/VolumeInPlaceEncryptor.java index 1a809091cc5..ffc32e73128 100644 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeInPlaceEncryptor.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeInPlaceEncryptor.java @@ -13,8 +13,11 @@ import org.zstack.header.errorcode.OperationFailureException; import org.zstack.header.keyprovider.EncryptedResourceKeyManager; import org.zstack.header.message.MessageReply; +import org.zstack.header.storage.primary.DeleteVolumeBitsOnPrimaryStorageMsg; import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageReply; import org.zstack.header.storage.primary.PrimaryStorageConstant; +import org.zstack.header.volume.VolumeFormat; import org.zstack.header.volume.VolumeVO; import org.zstack.storage.encrypt.VolumeEncryptedResourceKeyBackend; import org.zstack.storage.encrypt.VolumeEncryptedSecretHelper; @@ -24,7 +27,7 @@ import static org.zstack.core.Platform.operr; /** - * Performs an in-place LUKS conversion of an existing volume's bits. + * Performs a LUKS conversion of an existing volume's bits. * *

This is the single source of truth for the "encrypt-in-place" workflow that was * previously duplicated between {@link VolumeBase#handleMessage} (for the @@ -38,9 +41,9 @@ *

  • Materialize a DEK via {@link EncryptedResourceKeyManager#getOrCreateKey}.
    • *
  • Seal the DEK for the target host and pass it as {@code encryptedDek} * to the primary storage backend.
    • - *
  • Ask the primary storage backend to LUKS-convert the bits in place via - * {@code EncryptVolumeBitsOnPrimaryStorageMsg}. The kvmagent creates any - * single-use secret material file locally.
    • + *
  • Ask the primary storage backend to LUKS-convert the bits via + * {@code EncryptVolumeBitsOnPrimaryStorageMsg}. The backend may return a + * replacement install path.
    • *
  • Persist {@code VolumeVO.encrypted = true} after a successful conversion.
    • * * @@ -246,7 +249,7 @@ public void fail(ErrorCode err) { return; } - // 4) Ask the PS backend to LUKS-convert the bits in place. + // 4) Ask the PS backend to LUKS-convert the bits. EncryptVolumeBitsOnPrimaryStorageMsg emsg = new EncryptVolumeBitsOnPrimaryStorageMsg(); emsg.setPrimaryStorageUuid(psUuid); emsg.setHostUuid(ctx.getHostUuid()); @@ -262,17 +265,38 @@ public void run(MessageReply r) { r.getError(), completion); return; } + EncryptVolumeBitsOnPrimaryStorageReply reply = r.castReply(); + String newInstallPath = reply.getInstallPath(); + boolean installPathChanged = StringUtils.isNotBlank(newInstallPath) + && !StringUtils.equals(newInstallPath, installPath); + if (installPathChanged) { + volume.setInstallPath(newInstallPath); + } // 5) Persist encrypted=true. The short-circuit above guarantees we only // reach here when the row was previously encrypted=false; this is the // one and only place the flag flips, ensuring it always reflects the // on-disk reality. volume.setEncrypted(true); VolumeVO latest = dbf.updateAndRefresh(volume); + if (installPathChanged) { + deleteOldVolumeBitsBestEffort(psUuid, volume, installPath); + } completion.success(latest); } }); } + private void deleteOldVolumeBitsBestEffort(String psUuid, VolumeVO volume, String oldInstallPath) { + DeleteVolumeBitsOnPrimaryStorageMsg dmsg = new DeleteVolumeBitsOnPrimaryStorageMsg(); + dmsg.setPrimaryStorageUuid(psUuid); + dmsg.setInstallPath(oldInstallPath); + dmsg.setBitsUuid(volume.getUuid()); + dmsg.setBitsType(VolumeVO.class.getSimpleName()); + dmsg.setHypervisorType(VolumeFormat.getMasterHypervisorTypeByVolumeFormat(volume.getFormat()).toString()); + bus.makeTargetServiceIdByResourceUuid(dmsg, PrimaryStorageConstant.SERVICE_ID, psUuid); + bus.send(dmsg); + } + private void failAfterRollingBackMetadata(String volumeUuid, boolean attachedProviderHere, EncryptedResourceKeyManager.ResourceKeyResult keyResult, diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java b/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java index bf032d5df69..246279580db 100755 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java @@ -514,6 +514,9 @@ public void run(FlowTrigger trigger, Map data) { volumeInPlaceEncryptor.encryptInPlace(vol, ctx, new ReturnValueCompletion(trigger) { @Override public void success(VolumeVO latest) { + if (StringUtils.isNotBlank(latest.getInstallPath())) { + primaryStorageInstallPath = latest.getInstallPath(); + } trigger.next(); } diff --git a/test/src/test/groovy/org/zstack/storage/volume/VolumeManagerImplHostSelectionTest.groovy b/test/src/test/groovy/org/zstack/storage/volume/VolumeManagerImplHostSelectionTest.groovy index a6e1505cc81..b6b30b4bc6f 100644 --- a/test/src/test/groovy/org/zstack/storage/volume/VolumeManagerImplHostSelectionTest.groovy +++ b/test/src/test/groovy/org/zstack/storage/volume/VolumeManagerImplHostSelectionTest.groovy @@ -76,6 +76,21 @@ class VolumeManagerImplHostSelectionTest { Mockito.verify(hostQuery).add(HostVO_.uuid, SimpleQuery.Op.IN, Arrays.asList("host-connected")) } + @Test + void testDisconnectedPsHostRefsDoNotFallback() { + Mockito.when(clusterRefQuery.listValue()).thenReturn(Arrays.asList("cluster-uuid")) + Mockito.when(hostRefQuery.list()).thenReturn(Arrays.asList( + primaryStorageHostRef("host-disconnected-1", PrimaryStorageHostStatus.Disconnected), + primaryStorageHostRef("host-disconnected-2", PrimaryStorageHostStatus.Disconnected) + )) + + HostInventory inventory = invokeSelectHostForEncryptInPlace() + + assert inventory == null : "encrypt-in-place should not fallback to arbitrary cluster host when PS has only disconnected host refs" + Mockito.verify(hostQuery, Mockito.never()).add(HostVO_.uuid, SimpleQuery.Op.IN, Mockito.anyList()) + Mockito.verify(hostQuery, Mockito.never()).find() + } + private HostInventory invokeSelectHostForEncryptInPlace() { Method method = VolumeManagerImpl.class.getDeclaredMethod("selectHostForEncryptInPlace", String.class) method.accessible = true diff --git a/test/src/test/groovy/org/zstack/test/unittest/storage/encrypt/ZbsVolumeEncryptionExtensionTest.groovy b/test/src/test/groovy/org/zstack/test/unittest/storage/encrypt/ZbsVolumeEncryptionExtensionTest.groovy new file mode 100644 index 00000000000..7c997fc8d14 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/unittest/storage/encrypt/ZbsVolumeEncryptionExtensionTest.groovy @@ -0,0 +1,61 @@ +package org.zstack.test.unittest.storage.encrypt + +import org.junit.Before +import org.junit.Test +import org.mockito.Mockito +import org.zstack.header.storage.snapshot.VolumeSnapshotInventory +import org.zstack.header.volume.VolumeVO +import org.zstack.storage.encrypt.VolumeSnapshotEncryptionHelper +import org.zstack.storage.encrypt.ZbsVolumeEncryptionExtension + +import java.lang.reflect.Field + +class ZbsVolumeEncryptionExtensionTest { + private ZbsVolumeEncryptionExtension extension + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper + + @Before + void setUp() { + extension = new ZbsVolumeEncryptionExtension() + snapshotEncryptionHelper = Mockito.mock(VolumeSnapshotEncryptionHelper.class) + setField("snapshotEncryptionHelper", snapshotEncryptionHelper) + } + + @Test + void testSnapshotHooksDelegateForEncryptedVolume() { + VolumeVO volume = new VolumeVO() + volume.uuid = "volume-uuid" + volume.encrypted = true + + VolumeSnapshotInventory snapshot = new VolumeSnapshotInventory() + snapshot.uuid = "snapshot-uuid" + + extension.beforeTakeSnapshot("ps-uuid", volume, snapshot) + extension.afterTakeSnapshot("ps-uuid", volume, snapshot) + + Mockito.verify(snapshotEncryptionHelper).inheritVolumeKeyToSnapshot(volume, snapshot) + Mockito.verify(snapshotEncryptionHelper).completeTakeSnapshot(volume, snapshot) + } + + @Test + void testSnapshotHooksSkipPlainVolume() { + VolumeVO volume = new VolumeVO() + volume.uuid = "volume-uuid" + volume.encrypted = false + + VolumeSnapshotInventory snapshot = new VolumeSnapshotInventory() + snapshot.uuid = "snapshot-uuid" + + extension.beforeTakeSnapshot("ps-uuid", volume, snapshot) + extension.afterTakeSnapshot("ps-uuid", volume, snapshot) + + Mockito.verify(snapshotEncryptionHelper, Mockito.never()).inheritVolumeKeyToSnapshot(volume, snapshot) + Mockito.verify(snapshotEncryptionHelper, Mockito.never()).completeTakeSnapshot(volume, snapshot) + } + + private void setField(String name, Object value) { + Field field = ZbsVolumeEncryptionExtension.class.getDeclaredField(name) + field.accessible = true + field.set(extension, value) + } +}