diff --git a/PendingReleaseNotes b/PendingReleaseNotes index bc6068fa3310..c9afb25f7ed7 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -81,3 +81,13 @@ example.ver.1 > example.ver.2: additional key store is required. Note: attaching an encrypted RBD volume to a running Instance requires libvirt >= 10.1.0; booting an Instance from an encrypted RBD root disk works on older libvirt. + + * Added NVMe-over-Fabrics (TCP) support to the adaptive storage framework + and the Pure Storage FlashArray plugin. Volumes on a FlashArray primary + pool can now be delivered to KVM hypervisors over NVMe-TCP instead of + Fibre Channel by setting transport=nvme-tcp on the pool's provider URL. + Volumes are identified on the host via EUI-128 NGUIDs and attached to + guests as plain block devices through the native NVMe multipath layer; + no device-mapper multipath configuration is required. A new + Storage.StoragePoolType.NVMeTCP + MultipathNVMeOFAdapterBase / + NVMeTCPAdapter on the KVM side back the new pool type. diff --git a/api/src/main/java/com/cloud/storage/Storage.java b/api/src/main/java/com/cloud/storage/Storage.java index 275f6d25268d..42a18b9c83c5 100644 --- a/api/src/main/java/com/cloud/storage/Storage.java +++ b/api/src/main/java/com/cloud/storage/Storage.java @@ -185,7 +185,8 @@ public static enum StoragePoolType { Linstor(true, true, EncryptionSupport.Storage), DatastoreCluster(true, true, EncryptionSupport.Unsupported), // for VMware, to abstract pool of clusters StorPool(true, true, EncryptionSupport.Hypervisor), - FiberChannel(true, true, EncryptionSupport.Unsupported); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-) + FiberChannel(true, true, EncryptionSupport.Unsupported), // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-) + NVMeTCP(true, true, EncryptionSupport.Unsupported); // NVMe over TCP (NVMe-oF/TCP) Pool for KVM hypervisors; volumes are identified by EUI-128 NGUID (/dev/disk/by-id/nvme-eui.) private final boolean shared; private final boolean overProvisioning; diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 7674f1ce25a1..2a348cb058d1 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -616,8 +616,9 @@ private void handleVolumeMigrationFromManagedStorageToNonManagedStorage(VolumeIn private void verifyFormatWithPoolType(ImageFormat imageFormat, StoragePoolType poolType) { if (imageFormat != ImageFormat.VHD && imageFormat != ImageFormat.OVA && imageFormat != ImageFormat.QCOW2 && !(imageFormat == ImageFormat.RAW && (StoragePoolType.PowerFlex == poolType || - StoragePoolType.FiberChannel == poolType))) { - throw new CloudRuntimeException(String.format("Only the following image types are currently supported: %s, %s, %s, %s (for PowerFlex and FiberChannel)", + StoragePoolType.FiberChannel == poolType || + StoragePoolType.NVMeTCP == poolType))) { + throw new CloudRuntimeException(String.format("Only the following image types are currently supported: %s, %s, %s, %s (for PowerFlex, FiberChannel and NVMeTCP)", ImageFormat.VHD.toString(), ImageFormat.OVA.toString(), ImageFormat.QCOW2.toString(), ImageFormat.RAW.toString())); } } @@ -2796,7 +2797,8 @@ private void handleCreateTemplateFromManagedVolume(VolumeInfo volumeInfo, Templa if (!ImageFormat.QCOW2.equals(volumeInfo.getFormat()) && !(ImageFormat.RAW.equals(volumeInfo.getFormat()) && ( StoragePoolType.PowerFlex == storagePoolVO.getPoolType() || - StoragePoolType.FiberChannel == storagePoolVO.getPoolType()))) { + StoragePoolType.FiberChannel == storagePoolVO.getPoolType() || + StoragePoolType.NVMeTCP == storagePoolVO.getPoolType()))) { throw new CloudRuntimeException("When using managed storage, you can only create a template from a volume on KVM currently."); } @@ -2810,7 +2812,8 @@ private void handleCreateTemplateFromManagedVolume(VolumeInfo volumeInfo, Templa try { handleQualityOfServiceForVolumeMigration(volumeInfo, PrimaryDataStoreDriver.QualityOfServiceState.MIGRATION); - if (srcVolumeDetached || StoragePoolType.PowerFlex == storagePoolVO.getPoolType() || StoragePoolType.FiberChannel == storagePoolVO.getPoolType()) { + if (srcVolumeDetached || StoragePoolType.PowerFlex == storagePoolVO.getPoolType() || StoragePoolType.FiberChannel == storagePoolVO.getPoolType() + || StoragePoolType.NVMeTCP == storagePoolVO.getPoolType()) { _volumeService.grantAccess(volumeInfo, hostVO, srcDataStore); } @@ -2844,7 +2847,8 @@ private void handleCreateTemplateFromManagedVolume(VolumeInfo volumeInfo, Templa throw new CloudRuntimeException(msg + ex.getMessage(), ex); } finally { - if (srcVolumeDetached || StoragePoolType.PowerFlex == storagePoolVO.getPoolType() || StoragePoolType.FiberChannel == storagePoolVO.getPoolType()) { + if (srcVolumeDetached || StoragePoolType.PowerFlex == storagePoolVO.getPoolType() || StoragePoolType.FiberChannel == storagePoolVO.getPoolType() + || StoragePoolType.NVMeTCP == storagePoolVO.getPoolType()) { try { _volumeService.revokeAccess(volumeInfo, hostVO, srcDataStore); } @@ -2939,7 +2943,8 @@ private Map getSnapshotDetails(SnapshotInfo snapshotInfo) { long snapshotId = snapshotInfo.getId(); - if (storagePoolVO.getPoolType() == StoragePoolType.PowerFlex || storagePoolVO.getPoolType() == StoragePoolType.FiberChannel) { + if (storagePoolVO.getPoolType() == StoragePoolType.PowerFlex || storagePoolVO.getPoolType() == StoragePoolType.FiberChannel + || storagePoolVO.getPoolType() == StoragePoolType.NVMeTCP) { snapshotDetails.put(DiskTO.IQN, snapshotInfo.getPath()); } else { snapshotDetails.put(DiskTO.IQN, getSnapshotProperty(snapshotId, DiskTO.IQN)); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java index 20e3891478b4..32e756607e55 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java @@ -48,6 +48,7 @@ import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.hypervisor.kvm.storage.MultipathNVMeOFPool; import com.cloud.hypervisor.kvm.storage.MultipathSCSIPool; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; @@ -86,6 +87,10 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR return handleMultipathSCSIResize(command, pool); } + if (pool instanceof MultipathNVMeOFPool) { + return handleMultipathNVMeOFResize(command, pool); + } + if (spool.getType().equals(StoragePoolType.PowerFlex)) { pool.connectPhysicalDisk(volumeId, null); } @@ -239,4 +244,9 @@ private Answer handleMultipathSCSIResize(ResizeVolumeCommand command, KVMStorage ((MultipathSCSIPool)pool).resize(command.getPath(), command.getInstanceName(), command.getNewSize()); return new ResizeVolumeAnswer(command, true, ""); } + + private Answer handleMultipathNVMeOFResize(ResizeVolumeCommand command, KVMStoragePool pool) { + ((MultipathNVMeOFPool)pool).resize(command.getPath(), command.getInstanceName(), command.getNewSize()); + return new ResizeVolumeAnswer(command, true, ""); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 0cb32da318ea..29b23e6c6451 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -421,7 +421,8 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) { StoragePoolType.PowerFlex, StoragePoolType.Linstor, StoragePoolType.FiberChannel, - StoragePoolType.CLVM).contains(primaryPool.getType())) { + StoragePoolType.CLVM, + StoragePoolType.NVMeTCP).contains(primaryPool.getType())) { newTemplate.setFormat(ImageFormat.RAW); } else { newTemplate.setFormat(ImageFormat.QCOW2); @@ -454,7 +455,8 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) { public static String derivePath(PrimaryDataStoreTO primaryStore, DataTO destData, Map details) { String path = null; - if (primaryStore.getPoolType() == StoragePoolType.FiberChannel) { + if (primaryStore.getPoolType() == StoragePoolType.FiberChannel + || primaryStore.getPoolType() == StoragePoolType.NVMeTCP) { path = destData.getPath(); } else { path = details != null ? details.get("managedStoreTarget") : null; @@ -3513,7 +3515,8 @@ private Storage.ImageFormat getFormat(StoragePoolType poolType) { StoragePoolType.PowerFlex, StoragePoolType.Linstor, StoragePoolType.FiberChannel, - StoragePoolType.CLVM).contains(poolType)) { + StoragePoolType.CLVM, + StoragePoolType.NVMeTCP).contains(poolType)) { return ImageFormat.RAW; } else { return ImageFormat.QCOW2; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java new file mode 100644 index 000000000000..2128d96ccf1a --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFAdapterBase.java @@ -0,0 +1,722 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.storage; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.libvirt.LibvirtException; + +import com.cloud.storage.Storage; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import com.cloud.utils.storage.TemplateDownloaderUtil; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Base class for KVM storage adapters that surface remote block volumes over + * NVMe-over-Fabrics (NVMe-oF). It is the NVMe-oF counterpart of + * {@link MultipathSCSIAdapterBase}: it does not drive device-mapper multipath + * and does not rescan the SCSI bus, because NVMe-oF has its own multipath + * (the kernel's native NVMe multipath) and namespaces show up via + * asynchronous event notifications as soon as the target grants access. + * + * Volumes are identified on the host by their EUI-128 NGUID, which udev + * exposes as {@code /dev/disk/by-id/nvme-eui.}. + */ +public abstract class MultipathNVMeOFAdapterBase implements StorageAdaptor { + protected static Logger LOGGER = LogManager.getLogger(MultipathNVMeOFAdapterBase.class); + static final Map MapStorageUuidToStoragePool = new ConcurrentHashMap<>(); + + static final int DEFAULT_DISK_WAIT_SECS = 240; + static final long NS_RESCAN_TIMEOUT_SECS = 5; + private static final long POLL_INTERVAL_MS = 2000; + private static final long RESCAN_INTERVAL_MS = 10_000; + private static final long RESIZE_SETTLE_TIMEOUT_MS = 30_000; + + @Override + public KVMStoragePool getStoragePool(String uuid) { + // Dummy pool - adapters that dispatch per-volume don't need + // connectivity information on the pool itself. Use computeIfAbsent + // so concurrent callers do not race to create duplicate pool objects. + return MapStorageUuidToStoragePool.computeIfAbsent(uuid, u -> new MultipathNVMeOFPool(u, this)); + } + + @Override + public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { + return getStoragePool(uuid); + } + + public abstract String getName(); + + @Override + public abstract Storage.StoragePoolType getStoragePoolType(); + + public abstract boolean isStoragePoolTypeSupported(Storage.StoragePoolType type); + + /** + * Parse a {@code type=NVMETCP; address=; connid.=; ...} + * volume path and produce an {@link AddressInfo} with the host-side device + * path set to {@code /dev/disk/by-id/nvme-eui.}. + */ + public AddressInfo parseAndValidatePath(String inPath) { + if (inPath == null) { + throw new CloudRuntimeException("Cannot parse null volume path"); + } + String type = null; + String address = null; + String connectionId = null; + String path = null; + String hostname = resolveHostnameShort(); + String hostnameFq = resolveHostnameFq(); + String[] parts = inPath.split(";"); + for (String part : parts) { + // Cap the split at 2 so values containing '=' (e.g. base64) are not silently discarded. + String[] pair = part.split("=", 2); + if (pair.length != 2) { + continue; + } + String key = pair[0].trim(); + String value = pair[1].trim(); + if (key.equals("type")) { + type = value.toUpperCase(); + } else if (key.equals("address")) { + address = value; + } else if (key.equals("connid")) { + connectionId = value; + } else if (key.startsWith("connid.")) { + String inHostname = key.substring("connid.".length()); + if (inHostname.equals(hostname) || inHostname.equals(hostnameFq)) { + connectionId = value; + } + } + } + + if (!"NVMETCP".equals(type)) { + throw new CloudRuntimeException("Invalid address type provided for NVMe-oF target disk: " + type); + } + if (address == null) { + throw new CloudRuntimeException("NVMe-oF volume path is missing the required address field"); + } + path = "/dev/disk/by-id/nvme-eui." + address.toLowerCase(); + return new AddressInfo(type, address, connectionId, path); + } + + @Override + public KVMPhysicalDisk getPhysicalDisk(String volumePath, KVMStoragePool pool) { + if (StringUtils.isEmpty(volumePath) || pool == null) { + LOGGER.error("Unable to get physical disk, volume path or pool not specified"); + return null; + } + return getPhysicalDisk(parseAndValidatePath(volumePath), pool); + } + + private KVMPhysicalDisk getPhysicalDisk(AddressInfo address, KVMStoragePool pool) { + KVMPhysicalDisk disk = new KVMPhysicalDisk(address.getPath(), address.toString(), pool); + disk.setFormat(QemuImg.PhysicalDiskFormat.RAW); + + if (!isConnected(address.getPath())) { + if (!connectPhysicalDisk(address, pool, null)) { + throw new CloudRuntimeException("Unable to connect to NVMe namespace at " + address.getPath()); + } + } + long diskSize = getPhysicalDiskSize(address.getPath()); + disk.setSize(diskSize); + disk.setVirtualSize(diskSize); + return disk; + } + + @Override + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, Storage.StoragePoolType type, Map details, boolean isPrimaryStorage) { + LOGGER.info(String.format("createStoragePool(uuid,host,port,path,type) called with args (%s, %s, %d, %s, %s)", uuid, host, port, path, type)); + MultipathNVMeOFPool pool = new MultipathNVMeOFPool(uuid, host, port, path, type, details, this); + MapStorageUuidToStoragePool.put(uuid, pool); + return pool; + } + + @Override + public boolean deleteStoragePool(String uuid) { + MapStorageUuidToStoragePool.remove(uuid); + return true; + } + + @Override + public boolean deleteStoragePool(KVMStoragePool pool) { + return deleteStoragePool(pool.getUuid()); + } + + @Override + public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool, Map details, boolean isVMMigrate) { + if (StringUtils.isEmpty(volumePath) || pool == null) { + LOGGER.error("Unable to connect NVMe-oF physical disk: insufficient arguments"); + return false; + } + return connectPhysicalDisk(parseAndValidatePath(volumePath), pool, details); + } + + private boolean connectPhysicalDisk(AddressInfo address, KVMStoragePool pool, Map details) { + if (address.getConnectionId() == null) { + LOGGER.error("NVMe-oF volume " + address.getPath() + " on pool " + pool.getUuid() + " is missing a connid. token in its path"); + return false; + } + long waitSecs = DEFAULT_DISK_WAIT_SECS; + if (details != null && details.containsKey(com.cloud.storage.StorageManager.STORAGE_POOL_DISK_WAIT.toString())) { + String waitTime = details.get(com.cloud.storage.StorageManager.STORAGE_POOL_DISK_WAIT.toString()); + if (StringUtils.isNotEmpty(waitTime)) { + try { + waitSecs = Integer.parseInt(waitTime); + } catch (NumberFormatException e) { + LOGGER.warn("Ignoring non-numeric " + com.cloud.storage.StorageManager.STORAGE_POOL_DISK_WAIT.toString() + + "=[" + waitTime + "] on pool " + pool.getUuid() + ", falling back to default " + + DEFAULT_DISK_WAIT_SECS + "s"); + } + } + } + return waitForNamespace(address, pool, waitSecs); + } + + /** + * Poll for the EUI-keyed udev symlink to show up. On every iteration also + * nudge the kernel with {@code nvme ns-rescan} on every local NVMe + * controller, to cover arrays / firmware combinations that do not emit a + * reliable asynchronous event notification when a new namespace is + * mapped. + */ + private boolean waitForNamespace(AddressInfo address, KVMStoragePool pool, long waitSecs) { + if (waitSecs < 60) { + waitSecs = 60; + } + long deadline = System.currentTimeMillis() + (waitSecs * 1000); + File dev = new File(address.getPath()); + long lastRescan = 0; + while (System.currentTimeMillis() < deadline) { + if (dev.exists() && isConnected(address.getPath())) { + long size = getPhysicalDiskSize(address.getPath()); + if (size > 0) { + LOGGER.debug("Found NVMe namespace at " + address.getPath()); + return true; + } + } + // Throttle rescanAllControllers(): spawning one nvme ns-rescan per controller every + // 2s can be expensive on hosts with many controllers. RESCAN_INTERVAL_MS caps the + // rate; the first iteration still rescans immediately because lastRescan starts at 0. + long now = System.currentTimeMillis(); + if (now - lastRescan >= RESCAN_INTERVAL_MS) { + rescanAllControllers(); + lastRescan = now; + } + try { + Thread.sleep(POLL_INTERVAL_MS); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return false; + } + } + LOGGER.debug("NVMe namespace did not appear at " + address.getPath() + " within " + waitSecs + "s"); + return false; + } + + private void rescanAllControllers() { + try { + File sysClass = new File("/sys/class/nvme"); + File[] ctrls = sysClass.listFiles(); + if (ctrls == null) { + return; + } + for (File ctrl : ctrls) { + Process p = new ProcessBuilder("nvme", "ns-rescan", "/dev/" + ctrl.getName()) + .redirectErrorStream(true).start(); + if (!p.waitFor(NS_RESCAN_TIMEOUT_SECS, TimeUnit.SECONDS)) { + // Kill runaway nvme-cli invocations so they do not pile + // up under the JVM on every poll iteration while we + // are still waiting for the namespace to appear. + LOGGER.debug("nvme ns-rescan /dev/" + ctrl.getName() + + " did not complete within " + NS_RESCAN_TIMEOUT_SECS + + "s; terminating"); + p.destroyForcibly(); + } + } + } catch (Exception e) { + LOGGER.debug("nvme ns-rescan attempt failed: " + e.getMessage()); + } + } + + @Override + public boolean disconnectPhysicalDisk(String volumePath, KVMStoragePool pool) { + // NVMe-oF: the kernel drops the namespace as soon as the target removes + // this host's connection, so there is no host-side map to tear down the way + // Fibre Channel must flush its device-mapper entry. + return true; + } + + @Override + public boolean disconnectPhysicalDisk(Map volumeToDisconnect) { + return true; + } + + @Override + public boolean disconnectPhysicalDiskByPath(String localPath) { + // Same rationale as disconnectPhysicalDisk above. Only claim paths + // that look like NVMe EUI symlinks so we don't swallow foreign paths. + return localPath != null && localPath.startsWith("/dev/disk/by-id/nvme-eui."); + } + + @Override + public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.ImageFormat format) { + // Namespaces are created and destroyed by the storage provider, never from the + // host. Report "not handled here" instead of throwing, so a caller on a cleanup + // path behaves the same as it does on the Fibre Channel adapter. + LOGGER.info("deletePhysicalDisk({}) not handled by the NVMe-oF adapter; the storage provider owns namespace deletion", uuid); + return false; + } + + @Override + public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, PhysicalDiskFormat format, + Storage.ProvisioningType provisioningType, long size, byte[] passphrase) { + throw new UnsupportedOperationException("Unimplemented method 'createPhysicalDisk'"); + } + + @Override + public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, QemuImg.PhysicalDiskFormat format, long size, KVMStoragePool destPool) { + LOGGER.info("createTemplateFromDisk not supported on NVMe-oF pools"); + return null; + } + + @Override + public List listPhysicalDisks(String storagePoolUuid, KVMStoragePool pool) { + // The array owns the namespace inventory; it is not enumerable from the host. + LOGGER.info("listPhysicalDisks not supported on NVMe-oF pool {}", storagePoolUuid); + return null; + } + + @Override + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout) { + return copyPhysicalDisk(disk, name, destPool, timeout, null, null, null); + } + + /** + * Copy a template or source disk into a pre-provisioned NVMe namespace on + * this pool, so it can be consumed by a VM as a root or data volume. + * + * The destination namespace is expected to have already been created on + * the storage provider and connected to this host's hostgroup (that is + * the storage orchestrator's responsibility, not the KVM adapter's). All + * this method does is resolve the destination device path via + * {@link #getPhysicalDisk} - which will nvme ns-rescan and wait for the + * by-id/nvme-eui.<NGUID> symlink to show up if the kernel has not + * picked it up yet - and {@code qemu-img convert} the source image into + * the raw block device. + * + * User-space encryption passphrases are not supported: the provider + * already encrypts at rest and qemu-img LUKS on top of a shared + * hostgroup-scoped namespace is not a sensible layering. + */ + @Override + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, + byte[] srcPassphrase, byte[] destPassphrase, Storage.ProvisioningType provisioningType) { + if (disk == null || StringUtils.isEmpty(name) || destPool == null) { + throw new CloudRuntimeException("Unable to copy disk to NVMe-oF pool: source disk, destination volume name or destination pool not specified"); + } + if (srcPassphrase != null || destPassphrase != null) { + throw new CloudRuntimeException("NVMe-oF adapter does not support user-space encrypted source or destination volumes"); + } + + KVMPhysicalDisk destDisk = destPool.getPhysicalDisk(name); + if (destDisk == null || StringUtils.isEmpty(destDisk.getPath())) { + throw new CloudRuntimeException("Unable to resolve NVMe namespace for destination volume [" + name + "] on pool [" + destPool.getUuid() + "]"); + } + + destDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW); + destDisk.setVirtualSize(disk.getVirtualSize()); + destDisk.setSize(disk.getSize()); + + LOGGER.info(String.format("Copying source disk [path=%s, format=%s, virtualSize=%d] to NVMe-oF namespace [path=%s] on pool [%s]", + disk.getPath(), disk.getFormat(), disk.getVirtualSize(), destDisk.getPath(), destPool.getUuid())); + + QemuImgFile srcFile = new QemuImgFile(disk.getPath(), disk.getFormat()); + QemuImgFile destFile = new QemuImgFile(destDisk.getPath(), destDisk.getFormat()); + + try { + QemuImg qemu = new QemuImg(timeout); + qemu.convert(srcFile, destFile, true); + } catch (QemuImgException | LibvirtException e) { + throw new CloudRuntimeException("Failed to copy source disk [" + disk.getPath() + "] to NVMe-oF namespace [" + + destDisk.getPath() + "] on pool [" + destPool.getUuid() + "]: " + e.getMessage(), e); + } + + LOGGER.info("Successfully copied source disk to NVMe-oF namespace [" + destDisk.getPath() + "] on pool [" + destPool.getUuid() + "]"); + return destDisk; + } + + @Override + public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { + throw new UnsupportedOperationException("Unimplemented method 'createDiskFromTemplate'"); + } + + @Override + public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { + throw new UnsupportedOperationException("Unimplemented method 'createDiskFromTemplateBacking'"); + } + + /** + * Write a directly-downloaded template onto a namespace on this pool. + * + * The two path arguments are different kinds of thing, which is worth being explicit + * about: {@code templateFilePath} is a plain local file produced by the direct-download + * helper, while {@code destTemplatePath} is a managed volume path of the form + * {@code type=NVMETCP;address=...}. Only the destination may be resolved through + * {@link KVMStoragePool#getPhysicalDisk(String)}; passing the local file through it + * would hand a file name to {@link #parseAndValidatePath(String)} and fail. The caller + * has already issued connectPhysicalDisk() for the destination, so the namespace is + * expected to be present. + * + * The template is written as QCOW2 onto the raw namespace rather than as RAW. That + * matches the ScaleIO adaptor, is consistent with the QCOW2 format the template is + * registered with, and is what lets the Qcow2Inspector check the caller runs on the + * returned path succeed. + * + * Note what the caller does with the returned disk: KVMStorageProcessor puts + * {@code disk.getName()} into the DirectDownloadAnswer, and that becomes the template's + * install path and later its external name, which the provider interpolates into array + * REST calls. {@link #getPhysicalDisk(String, KVMStoragePool)} names a disk + * {@code AddressInfo.toString()}, which contains spaces and brackets and would produce + * a name that cannot be placed in a URI. So the disk handed back here is named with the + * managed volume path we were given, matching what a volume records. + */ + @Override + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { + if (StringUtils.isAnyEmpty(templateFilePath, destTemplatePath) || destPool == null) { + throw new CloudRuntimeException("Unable to create a template from a direct download file on an NVMe-oF pool: " + + "template file path, destination template path or destination pool not specified"); + } + + if (!Storage.ImageFormat.QCOW2.equals(format) && !Storage.ImageFormat.RAW.equals(format)) { + throw new CloudRuntimeException("Unsupported direct download template format for NVMe-oF pools: " + format + + "; expected " + Storage.ImageFormat.QCOW2 + " or " + Storage.ImageFormat.RAW); + } + + File sourceFile = new File(templateFilePath); + if (!sourceFile.exists()) { + throw new CloudRuntimeException("Direct download template file " + templateFilePath + " does not exist on this host"); + } + + LOGGER.debug("Creating a template on NVMe-oF pool [{}] from direct download file [{}] into [{}], format [{}]", + destPool.getUuid(), templateFilePath, destTemplatePath, format); + + String srcTemplateFilePath = templateFilePath; + KVMPhysicalDisk destDisk; + try { + destDisk = destPool.getPhysicalDisk(destTemplatePath); + if (destDisk == null || StringUtils.isEmpty(destDisk.getPath())) { + throw new CloudRuntimeException("Unable to resolve the NVMe namespace for destination template path [" + + destTemplatePath + "] on pool [" + destPool.getUuid() + "]"); + } + + // Direct-download templates are commonly published compressed. + if (TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) { + srcTemplateFilePath = sourceFile.getParent() + "/" + UUID.randomUUID().toString(); + LOGGER.debug("Extracting downloaded template [{}] to [{}]", templateFilePath, srcTemplateFilePath); + Script.runSimpleBashScript(TemplateDownloaderUtil.getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath)); + Script.runSimpleBashScript("rm -f " + templateFilePath); + } + + QemuImg.PhysicalDiskFormat srcFormat = Storage.ImageFormat.RAW.equals(format) + ? QemuImg.PhysicalDiskFormat.RAW : QemuImg.PhysicalDiskFormat.QCOW2; + + QemuImg qemu = new QemuImg(timeout); + QemuImgFile srcFile = new QemuImgFile(srcTemplateFilePath, srcFormat); + // Populates the virtual size, and fails early if the file is unreadable or is + // not in the format the template claims to be. + qemu.info(srcFile); + + long namespaceSize = getPhysicalDiskSize(destDisk.getPath()); + if (namespaceSize > 0 && srcFile.getSize() > namespaceSize) { + throw new CloudRuntimeException("Direct download template needs " + srcFile.getSize() + + " bytes but the NVMe namespace at " + destDisk.getPath() + " is only " + namespaceSize + " bytes"); + } + + QemuImgFile destFile = new QemuImgFile(destDisk.getPath(), QemuImg.PhysicalDiskFormat.QCOW2); + destFile.setSize(srcFile.getSize()); + + LOGGER.debug("Converting [{}] onto NVMe namespace [{}]", srcFile.getFileName(), destDisk.getPath()); + qemu.create(destFile); + qemu.convert(srcFile, destFile); + + KVMPhysicalDisk template = new KVMPhysicalDisk(destDisk.getPath(), destTemplatePath, destPool); + template.setFormat(QemuImg.PhysicalDiskFormat.QCOW2); + template.setVirtualSize(srcFile.getSize()); + template.setSize(srcFile.getSize()); + destDisk = template; + LOGGER.info("Wrote direct download template onto NVMe namespace [{}] on pool [{}]", + destDisk.getPath(), destPool.getUuid()); + } catch (QemuImgException | LibvirtException e) { + throw new CloudRuntimeException("Failed to write the direct download template [" + templateFilePath + + "] onto the NVMe namespace for [" + destTemplatePath + "] on pool [" + destPool.getUuid() + + "]: " + e.getMessage(), e); + } finally { + // Only remove what we extracted; the original download belongs to the caller. + if (!srcTemplateFilePath.equals(templateFilePath)) { + Script.runSimpleBashScript("rm -f " + srcTemplateFilePath); + } + } + + return destDisk; + } + + @Override + public boolean refresh(KVMStoragePool pool) { + return true; + } + + @Override + public boolean createFolder(String uuid, String path) { + return createFolder(uuid, path, null); + } + + @Override + public boolean createFolder(String uuid, String path, String localPath) { + // Block storage has no directory structure to create. Succeed rather than + // throw, matching the Fibre Channel adapter. + LOGGER.info("createFolder({}, {}, {}) is a no-op on NVMe-oF pools", uuid, path, localPath); + return true; + } + + /** + * Host-side half of a volume resize. The storage provider has already grown the + * namespace on the array by the time we get here, so all that remains is to make + * the new capacity visible locally and tell a running guest about it. + * + * Unlike the SCSI/FC path there is no device-mapper map to grow: the kernel picks + * up the new namespace size either from the target's asynchronous event + * notification or from an explicit {@code nvme ns-rescan}, which we issue here + * rather than waiting for the AEN. + */ + public void resize(String path, String vmName, long newSize) { + AddressInfo address = parseAndValidatePath(path); + if (address == null || address.getPath() == null) { + throw new CloudRuntimeException("Unable to resize NVMe-oF volume, could not derive a device path from [" + path + "]"); + } + + LOGGER.debug("Resizing NVMe-oF volume " + address.getPath() + " to " + newSize + " bytes for VM " + vmName); + + rescanAllControllers(); + + long observed = waitForNamespaceSize(address.getPath(), newSize); + if (observed < newSize) { + // Not fatal: the array has already been grown, and the kernel may still + // catch up via an AEN. Surface it rather than failing the operation, so + // the management server does not roll back a resize that did happen. + LOGGER.warn("NVMe namespace " + address.getPath() + " still reports " + observed + + " bytes after rescan, expected at least " + newSize + + "; the guest may not observe the new size until the next rescan"); + } + + notifyGuestOfResize(address.getPath(), vmName, newSize, address.getAddress()); + } + + /** + * Poll the block device until it reports at least {@code expectedSize}, since + * ns-rescan and AEN processing are asynchronous. + * + * @return the last size observed, which may be smaller than expected on timeout. + */ + private long waitForNamespaceSize(String devicePath, long expectedSize) { + long deadline = System.currentTimeMillis() + RESIZE_SETTLE_TIMEOUT_MS; + long observed = getPhysicalDiskSize(devicePath); + while (observed < expectedSize && System.currentTimeMillis() < deadline) { + try { + Thread.sleep(POLL_INTERVAL_MS); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + break; + } + rescanAllControllers(); + observed = getPhysicalDiskSize(devicePath); + } + return observed; + } + + /** + * Ask libvirt to re-read the size of the guest's block device, so a running VM + * sees the extra capacity without a reboot. A stopped VM needs nothing here: it + * picks up the new size when the disk is next attached. + */ + private void notifyGuestOfResize(String devicePath, String vmName, long newSize, String eui) { + if (StringUtils.isEmpty(vmName)) { + LOGGER.debug("No VM name supplied for resize of " + devicePath + "; skipping guest notification"); + return; + } + + if (!isVmRunning(vmName)) { + LOGGER.debug("VM " + vmName + " is not running; skipping guest notification for " + devicePath); + return; + } + + String target = findDomainDiskTarget(vmName, devicePath, eui); + if (target == null) { + LOGGER.warn("Could not find a disk target for " + devicePath + " in domain " + vmName + + "; skipping guest notification"); + return; + } + + // virsh blockresize takes the new size in KiB. + Script cmd = new Script("virsh", LOGGER); + cmd.add("blockresize"); + cmd.add("--path", target); + cmd.add("--size", String.valueOf(newSize / 1024L)); + cmd.add(vmName); + String result = cmd.execute(); + if (result != null) { + LOGGER.warn("virsh blockresize of " + target + " on " + vmName + " failed: " + result); + } else { + LOGGER.info("Notified " + vmName + " of new size " + newSize + " bytes for " + target); + } + } + + private boolean isVmRunning(String vmName) { + Script cmd = new Script("virsh", LOGGER); + cmd.add("domstate", vmName); + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String result = cmd.execute(parser); + return result == null && parser.getLine() != null && parser.getLine().trim().startsWith("running"); + } + + /** + * Resolve the domain-local disk target (vda, vdb, ...) backing {@code devicePath}. + * + * libvirt reports the source as it was configured, but may instead surface a + * canonicalised /dev/nvmeXnY in place of the /dev/disk/by-id symlink we attached, + * so accept either form, falling back to matching the bare EUI. + */ + private String findDomainDiskTarget(String vmName, String devicePath, String eui) { + Script cmd = new Script("virsh", LOGGER); + cmd.add("domblklist", vmName); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = cmd.execute(parser); + if (result != null || parser.getLines() == null) { + return null; + } + String canonical = resolveCanonicalPath(devicePath); + for (String line : parser.getLines().split("\\R")) { + String[] cols = line.trim().split("\\s+"); + if (cols.length < 2) { + continue; + } + String source = cols[1]; + if (source.equals(devicePath) + || (canonical != null && source.equals(canonical)) + || (StringUtils.isNotEmpty(eui) && source.toLowerCase().contains(eui.toLowerCase()))) { + return cols[0]; + } + } + return null; + } + + /** Resolve a /dev/disk/by-id symlink to its /dev/nvmeXnY target, or null. */ + private String resolveCanonicalPath(String devicePath) { + try { + return new File(devicePath).getCanonicalPath(); + } catch (Exception e) { + LOGGER.debug("Could not canonicalise " + devicePath + ": " + e.getMessage()); + return null; + } + } + + boolean isConnected(String path) { + Script test = new Script("/bin/test", LOGGER); + test.add("-b", path); + test.execute(); + return test.getExitValue() == 0; + } + + long getPhysicalDiskSize(String diskPath) { + if (StringUtils.isEmpty(diskPath)) { + return 0; + } + Script cmd = new Script("blockdev", LOGGER); + cmd.add("--getsize64", diskPath); + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String result = cmd.execute(parser); + if (result != null) { + LOGGER.debug("Unable to get the disk size at path: " + diskPath); + return 0; + } + try { + return Long.parseLong(parser.getLine()); + } catch (NumberFormatException e) { + return 0; + } + } + + private static String resolveHostnameShort() { + try { + String h = java.net.InetAddress.getLocalHost().getHostName(); + int dot = h.indexOf('.'); + return dot > 0 ? h.substring(0, dot) : h; + } catch (Exception e) { + return null; + } + } + + private static String resolveHostnameFq() { + try { + return java.net.InetAddress.getLocalHost().getCanonicalHostName(); + } catch (Exception e) { + return null; + } + } + + /** + * Same shape as {@link MultipathSCSIAdapterBase.AddressInfo}. Kept + * separate so this class can be consumed by adapters that don't share the + * SCSI base. + */ + public static final class AddressInfo { + String type; + String address; + String connectionId; + String path; + + public AddressInfo(String type, String address, String connectionId, String path) { + this.type = type; + this.address = address; + this.connectionId = connectionId; + this.path = path; + } + + public String getType() { return type; } + public String getAddress() { return address; } + public String getConnectionId() { return connectionId; } + public String getPath() { return path; } + + public String toString() { + return String.format("AddressInfo %s [address=%s, connectionId=%s, path=%s]", type, address, connectionId, path); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFPool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFPool.java new file mode 100644 index 000000000000..fec1e28f404c --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFPool.java @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.storage; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.joda.time.Duration; + +import com.cloud.agent.api.to.HostTO; +import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; +import com.cloud.storage.Storage; +import com.cloud.storage.Storage.ProvisioningType; + +/** + * KVMStoragePool for NVMe-over-Fabrics pools. Mirror of + * {@link MultipathSCSIPool} for adapters based on + * {@link MultipathNVMeOFAdapterBase}. Every data operation is delegated + * back to the adapter; the pool itself only tracks addressing/identity. + */ +public class MultipathNVMeOFPool implements KVMStoragePool { + private final String uuid; + private final String sourceHost; + private final int sourcePort; + private final String sourceDir; + private final Storage.StoragePoolType storagePoolType; + private final StorageAdaptor storageAdaptor; + private final Map details; + private long capacity; + private long used; + private long available; + + public MultipathNVMeOFPool(String uuid, String host, int port, String path, + Storage.StoragePoolType poolType, Map poolDetails, StorageAdaptor adaptor) { + this.uuid = uuid; + this.sourceHost = host; + this.sourcePort = port; + this.sourceDir = path; + this.storagePoolType = poolType; + this.storageAdaptor = adaptor; + this.details = poolDetails; + this.capacity = 0; + this.used = 0; + this.available = 0; + } + + public MultipathNVMeOFPool(String uuid, StorageAdaptor adaptor) { + this.uuid = uuid; + this.sourceHost = null; + this.sourcePort = -1; + this.sourceDir = null; + this.storagePoolType = Storage.StoragePoolType.NVMeTCP; + this.storageAdaptor = adaptor; + this.details = new HashMap<>(); + this.capacity = 0; + this.used = 0; + this.available = 0; + } + + @Override + public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, ProvisioningType provisioningType, long size, byte[] passphrase) { + return null; + } + + @Override + public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, ProvisioningType provisioningType, long size, byte[] passphrase) { + return null; + } + + @Override + public boolean connectPhysicalDisk(String volumeUuid, Map details) { + return storageAdaptor.connectPhysicalDisk(volumeUuid, this, details, false); + } + + @Override + public KVMPhysicalDisk getPhysicalDisk(String volumeId) { + return storageAdaptor.getPhysicalDisk(volumeId, this); + } + + @Override + public boolean disconnectPhysicalDisk(String volumeUuid) { + return storageAdaptor.disconnectPhysicalDisk(volumeUuid, this); + } + + @Override + public boolean deletePhysicalDisk(String volumeUuid, Storage.ImageFormat format) { + return true; + } + + /** + * Host-side resize, mirroring {@link MultipathSCSIPool#resize}. The provider has + * already grown the volume on the array; the adaptor makes it visible here. + */ + public void resize(String path, String vmName, long newSize) { + ((MultipathNVMeOFAdapterBase)storageAdaptor).resize(path, vmName, newSize); + } + + @Override + public List listPhysicalDisks() { + return null; + } + + @Override + public String getUuid() { + return uuid; + } + + public void setCapacity(long capacity) { this.capacity = capacity; } + @Override public long getCapacity() { return this.capacity; } + public void setUsed(long used) { this.used = used; } + @Override public long getUsed() { return this.used; } + public void setAvailable(long available) { this.available = available; } + @Override public long getAvailable() { return this.available; } + + @Override public boolean refresh() { return false; } + @Override public boolean isExternalSnapshot() { return true; } + @Override public String getLocalPath() { return null; } + @Override public String getSourceHost() { return this.sourceHost; } + @Override public String getSourceDir() { return this.sourceDir; } + @Override public int getSourcePort() { return this.sourcePort; } + @Override public String getAuthUserName() { return null; } + @Override public String getAuthSecret() { return null; } + @Override public Storage.StoragePoolType getType() { return storagePoolType; } + @Override public boolean delete() { return false; } + @Override public QemuImg.PhysicalDiskFormat getDefaultFormat() { return QemuImg.PhysicalDiskFormat.RAW; } + @Override public boolean createFolder(String path) { return false; } + @Override public boolean supportsConfigDriveIso() { return false; } + @Override public Map getDetails() { return this.details; } + @Override public boolean isPoolSupportHA() { return false; } + @Override public String getHearthBeatPath() { return null; } + + @Override + public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { + return null; + } + + @Override public String getStorageNodeId() { return null; } + + @Override + public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { return null; } + + @Override + public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, + String volumeUUIDListString, String vmActivityCheckPath, long duration) { + return null; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java new file mode 100644 index 000000000000..596f5d7bb9c5 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.storage; + +import com.cloud.storage.Storage; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * StorageAdaptor for the {@link Storage.StoragePoolType#NVMeTCP} pool type. + * All operational logic lives in {@link MultipathNVMeOFAdapterBase}; this + * class just binds that logic to a pool type so + * {@link KVMStoragePoolManager} can find it via reflection. + */ +public class NVMeTCPAdapter extends MultipathNVMeOFAdapterBase { + private static final Logger LOGGER = LogManager.getLogger(NVMeTCPAdapter.class); + + public NVMeTCPAdapter() { + LOGGER.info("Loaded NVMeTCPAdapter for StorageLayer"); + } + + @Override + public String getName() { + return "NVMeTCPAdapter"; + } + + @Override + public Storage.StoragePoolType getStoragePoolType() { + return Storage.StoragePoolType.NVMeTCP; + } + + @Override + public boolean isStoragePoolTypeSupported(Storage.StoragePoolType type) { + return Storage.StoragePoolType.NVMeTCP.equals(type); + } +} diff --git a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java index 25577903e3d8..a954560508c4 100644 --- a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java +++ b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java @@ -35,6 +35,7 @@ public interface ProviderVolume { public String getExternalName(); public String getExternalConnectionId(); public enum AddressType { - FIBERWWN + FIBERWWN, + NVMETCP } } diff --git a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java index c8efc08c2892..4714576498ab 100644 --- a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java @@ -179,7 +179,7 @@ public DataStore initialize(Map dsInfos) { parameters.setHost(uri.getHost()); parameters.setPort(uri.getPort()); parameters.setPath(uri.getPath() + "?" + uri.getQuery()); - parameters.setType(StoragePoolType.FiberChannel); + parameters.setType(pickPoolType(uri)); parameters.setZoneId(zoneId); parameters.setPodId(podId); parameters.setClusterId(clusterId); @@ -402,4 +402,26 @@ public void disableStoragePool(DataStore store) { logger.info("Disabling storage pool {}", store); _dataStoreHelper.disable(store); } + + /** + * Resolve the CloudStack StoragePoolType from the provider URL. Adaptive + * plugins advertise the underlying fabric via a {@code transport=} query + * parameter on the URL; when absent we keep the legacy FiberChannel + * default for backwards compatibility with adapters that still assume it. + */ + private static StoragePoolType pickPoolType(java.net.URL uri) { + String query = uri.getQuery(); + if (query != null) { + for (String tok : query.split("&")) { + int i = tok.indexOf('='); + if (i > 0 && "transport".equalsIgnoreCase(tok.substring(0, i))) { + String value = tok.substring(i + 1); + if ("nvme-tcp".equalsIgnoreCase(value)) { + return StoragePoolType.NVMeTCP; + } + } + } + } + return StoragePoolType.FiberChannel; + } } diff --git a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java index fae94bb1bea0..4e12e3eb18d6 100644 --- a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java +++ b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayAdapter.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; @@ -78,6 +79,9 @@ public class FlashArrayAdapter implements ProviderAdapter { public static final String HOSTGROUP = "hostgroup"; public static final String STORAGE_POD = "pod"; + public static final String TRANSPORT = "transport"; + public static final String TRANSPORT_FC = "fc"; + public static final String TRANSPORT_NVME_TCP = "nvme-tcp"; public static final String KEY_TTL = "keyttl"; public static final String CONNECT_TIMEOUT_MS = "connectTimeoutMs"; public static final String POST_COPY_WAIT_MS = "postCopyWaitMs"; @@ -90,6 +94,9 @@ public class FlashArrayAdapter implements ProviderAdapter { private static final String API_LOGIN_VERSION_DEFAULT = "1.19"; private static final String API_VERSION_DEFAULT = "2.23"; + /** A FlashArray NVMe namespace EUI-128 is exactly 32 hexadecimal characters. */ + private static final Pattern EUI128_PATTERN = Pattern.compile("[0-9a-fA-F]{32}"); + // URLs for which the legacy-auth deprecation WARN has already been emitted, // so we don't spam the logs once per refresh per pool while it's still configured. private static final Set WARNED_LEGACY_URLS = ConcurrentHashMap.newKeySet(); @@ -99,7 +106,7 @@ public class FlashArrayAdapter implements ProviderAdapter { public String hostgroup = null; private static final DateTimeFormatter DELETION_TIMESTAMP_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHHmmss").withZone(ZoneOffset.UTC); - + private AddressType volumeAddressType = AddressType.FIBERWWN; private String username; private String password; private String accessToken; @@ -133,13 +140,14 @@ public ProviderVolume create(ProviderAdapterContext context, ProviderAdapterData request, new TypeReference>() { }); - return (ProviderVolume) getFlashArrayItem(list); + return withAddressType((FlashArrayVolume) getFlashArrayItem(list)); } /** - * Volumes must be added to a host set to be visable to the hosts. - * the Hostset should contain all the hosts that are membrers of the zone or - * cluster (depending on Cloudstack Storage Pool configuration) + * Connect the volume to the host that was granted access. The storage framework + * grants and revokes access one host at a time, so each host is given its own + * connection: a host-group scoped connection is shared by every member host and + * therefore cannot express the removal of a single host's access. */ @Override public String attach(ProviderAdapterContext context, ProviderAdapterDataObject dataObject, String hostname) { @@ -160,14 +168,21 @@ public String attach(ProviderAdapterContext context, ProviderAdapterDataObject d } if (list == null || list.getItems() == null || list.getItems().size() == 0) { - throw new RuntimeException("Volume attach did not return lun information"); + throw new RuntimeException("Volume attach did not return connection information " + + "(expected lun for Fibre Channel or nsid for NVMe-TCP)"); } FlashArrayConnection connection = (FlashArrayConnection) this.getFlashArrayItem(list); + if (AddressType.NVMETCP.equals(volumeAddressType)) { + // The FlashArray REST API does not return nsid in the connections + // payload for NVMe-TCP. The namespace is identified on the host by + // EUI-128 (see FlashArrayVolume.getAddress()); the value returned + // here is stored by the driver only for informational purposes. + return connection.getNsid() != null ? "" + connection.getNsid() : "1"; + } if (connection.getLun() == null) { throw new RuntimeException("Volume attach missing lun field"); } - return "" + connection.getLun(); } catch (Throwable e) { @@ -179,15 +194,35 @@ public String attach(ProviderAdapterContext context, ProviderAdapterDataObject d }); if (list != null && list.getItems() != null) { for (FlashArrayConnection conn : list.getItems()) { - if (conn.getHost() != null && conn.getHost().getName() != null && - (conn.getHost().getName().equals(hostname) || conn.getHost().getName().equals(hostname.substring(0, hostname.indexOf('.')))) && - conn.getLun() != null) { + if (AddressType.NVMETCP.equals(volumeAddressType)) { + // Match the host-scoped connection this adapter creates. A + // host-group scoped match is kept only as a fallback, so that + // volumes still carrying a group connection created by an + // earlier release continue to resolve here. + if (conn.getHost() != null && conn.getHost().getName() != null + && (conn.getHost().getName().equals(hostname) + || (hostname.indexOf('.') > 0 + && conn.getHost().getName() + .equals(hostname.substring(0, hostname.indexOf('.')))))) { + return conn.getNsid() != null ? "" + conn.getNsid() : "1"; + } + if (hostgroup != null && conn.getHostGroup() != null + && conn.getHostGroup().getName() != null + && conn.getHostGroup().getName().equals(hostgroup)) { + return conn.getNsid() != null ? "" + conn.getNsid() : "1"; + } + } else if (conn.getHost() != null && conn.getHost().getName() != null + && (conn.getHost().getName().equals(hostname) + || (hostname.indexOf('.') > 0 + && conn.getHost().getName() + .equals(hostname.substring(0, hostname.indexOf('.'))))) + && conn.getLun() != null) { return "" + conn.getLun(); } } - throw new RuntimeException("Volume lun is not found in existing connection"); + throw new RuntimeException("Volume connection identifier (lun/nsid) not found in existing connection"); } else { - throw new RuntimeException("Volume lun is not found in existing connection"); + throw new RuntimeException("Volume connection is not found in existing connection list"); } } else { throw e; @@ -198,12 +233,10 @@ public String attach(ProviderAdapterContext context, ProviderAdapterDataObject d @Override public void detach(ProviderAdapterContext context, ProviderAdapterDataObject dataObject, String hostname) { String volumeName = normalizeName(pod, dataObject.getExternalName()); - // hostname is always provided by cloudstack, but we will detach from hostgroup - // if this pool is configured to use hostgroup for attachments - if (hostgroup != null) { - DELETE("/connections?host_group_names=" + hostgroup + "&volume_names=" + volumeName); - } - + // Only the connection for this host may be removed. A host-group scoped + // connection is shared by every member host, so deleting it here would revoke + // the volume from all of them -- including the host a live-migrating VM has + // just moved to, which pulls the namespace out from under the running guest. FlashArrayHost host = getHost(hostname); if (host != null) { DELETE("/connections?host_names=" + host.getName() + "&volume_names=" + volumeName); @@ -307,14 +340,37 @@ public ProviderVolume getVolumeByAddress(ProviderAdapterContext context, Address throw new RuntimeException("Invalid search criteria provided for getVolumeByAddress"); } - // only support WWN type addresses at this time. - if (!ProviderVolume.AddressType.FIBERWWN.equals(addressType)) { + String serial; + if (ProviderVolume.AddressType.FIBERWWN.equals(addressType)) { + // Strip the NAA prefix (1 char) + Pure OUI to recover the volume serial. + serial = address.substring(FlashArrayVolume.PURE_OUI.length() + 1).toUpperCase(); + } else if (ProviderVolume.AddressType.NVMETCP.equals(addressType)) { + // Reverse the EUI-128 layout: serial = eui[2:16] + eui[22:32], after + // stripping the optional "eui." prefix that appears in udev paths. + String eui = address.startsWith("eui.") ? address.substring(4) : address; + if (eui == null || !EUI128_PATTERN.matcher(eui).matches()) { + throw new RuntimeException("Invalid NVMe-TCP EUI-128 address [" + + address + "]: expected 32 hexadecimal characters, got " + + (eui == null ? "null" : String.valueOf(eui.length()))); + } + // Validate the FlashArray EUI-128 layout before deriving a serial from it, so a + // malformed or tampered address cannot be mapped onto an unintended volume: + // 00 + serial[0:14] + + serial[14:24] + if (!eui.startsWith("00")) { + throw new RuntimeException("Invalid NVMe-TCP EUI-128 address [" + address + + "]: expected a \"00\" prefix for a FlashArray namespace"); + } + if (!eui.regionMatches(true, 16, FlashArrayVolume.PURE_OUI_EUI, 0, + FlashArrayVolume.PURE_OUI_EUI.length())) { + throw new RuntimeException("Invalid NVMe-TCP EUI-128 address [" + address + + "]: expected the Pure Storage OUI [" + FlashArrayVolume.PURE_OUI_EUI + + "] at offset 16"); + } + serial = (eui.substring(2, 16) + eui.substring(22)).toUpperCase(); + } else { throw new RuntimeException( "Invalid volume address type [" + addressType + "] requested for volume search"); } - - // convert WWN to serial to search on. strip out WWN type # + Flash OUI value - String serial = address.substring(FlashArrayVolume.PURE_OUI.length() + 1).toUpperCase(); String query = "serial='" + serial + "'"; FlashArrayVolume volume = null; @@ -328,7 +384,7 @@ public ProviderVolume getVolumeByAddress(ProviderAdapterContext context, Address return null; } - volume = (FlashArrayVolume) this.getFlashArrayItem(list); + volume = withAddressType((FlashArrayVolume) this.getFlashArrayItem(list)); if (volume != null && volume.getAddress() == null) { return null; } @@ -365,8 +421,11 @@ public ProviderSnapshot snapshot(ProviderAdapterContext context, ProviderAdapter "/volume-snapshots?source_names=" + sourceDataObject.getExternalName(), null, new TypeReference>() { }); - - return (FlashArrayVolume) getFlashArrayItem(list); + // Stamp the pool's volume address type so ProviderSnapshot.getAddress() + // emits an NVMe EUI-128 on NVMe-TCP pools. Without this, the adaptive + // driver persists the snapshot with an FC-style WWN and subsequent + // revert/list operations cannot locate the namespace. + return withAddressType((FlashArrayVolume) getFlashArrayItem(list)); } /** @@ -409,7 +468,12 @@ public ProviderSnapshot getSnapshot(ProviderAdapterContext context, ProviderAdap "/volume-snapshots?names=" + dataObject.getExternalName(), new TypeReference>() { }); - return (FlashArrayVolume) getFlashArrayItem(list); + // Stamp the pool's volume address type so ProviderSnapshot.getAddress() + // emits an NVMe EUI-128 on NVMe-TCP pools instead of the FIBERWWN + // default. Without this, the adaptive driver persists the snapshot + // path with an FC-style WWN and revert/list fails to locate the + // namespace on the host. + return withAddressType((FlashArrayVolume) getFlashArrayItem(list)); } @Override @@ -437,10 +501,7 @@ public ProviderVolume copy(ProviderAdapterContext context, ProviderAdapterDataOb if (sourceDataObject.getType().equals(ProviderAdapterDataObject.Type.SNAPSHOT)) { currentVol = getSnapshot(sourceDataObject.getExternalName()); } else { - currentVol = (FlashArrayVolume) this - .getFlashArrayItem(GET("/volumes?names=" + sourceDataObject.getExternalName(), - new TypeReference>() { - })); + currentVol = getVolume(sourceDataObject.getExternalName()); } if (currentVol == null) { @@ -459,7 +520,7 @@ public ProviderVolume copy(ProviderAdapterContext context, ProviderAdapterDataOb "/volumes?names=" + payload.getExternalName() + "&overwrite=true", payload, new TypeReference>() { }); - FlashArrayVolume outVolume = (FlashArrayVolume) getFlashArrayItem(list); + FlashArrayVolume outVolume = withAddressType((FlashArrayVolume) getFlashArrayItem(list)); pause(postCopyWait); return outVolume; } @@ -764,6 +825,13 @@ private void login() { } } + String transport = connectionDetails.get(FlashArrayAdapter.TRANSPORT); + if (transport == null) { + transport = queryParms.get(FlashArrayAdapter.TRANSPORT); + } + volumeAddressType = TRANSPORT_NVME_TCP.equalsIgnoreCase(transport) + ? AddressType.NVMETCP : AddressType.FIBERWWN; + // retrieve for legacy purposes. if set, we'll remove any connections to hostgroup we find and use the host hostgroup = connectionDetails.get(FlashArrayAdapter.HOSTGROUP); if (hostgroup == null) { @@ -919,11 +987,17 @@ private void removeVlunsAll(ProviderAdapterContext context, String volumeNamespa } } + /** + * Look up a volume by name. The pool's address type is always stamped onto the + * result, so no caller can emit an FC-style WWN for a volume that lives on an + * NVMe-TCP pool (see {@link FlashArrayVolume#getAddress()}). The snapshot + * accessor below holds the same invariant. + */ private FlashArrayVolume getVolume(String volumeName) { FlashArrayList list = GET("/volumes?names=" + volumeName, new TypeReference>() { }); - return (FlashArrayVolume) getFlashArrayItem(list); + return withAddressType((FlashArrayVolume) getFlashArrayItem(list)); } private FlashArrayPod getVolumeNamespace(String name) { @@ -937,7 +1011,14 @@ private FlashArrayVolume getSnapshot(String snapshotName) { FlashArrayList list = GET("/volume-snapshots?names=" + snapshotName, new TypeReference>() { }); - return (FlashArrayVolume) getFlashArrayItem(list); + return withAddressType((FlashArrayVolume) getFlashArrayItem(list)); + } + + private FlashArrayVolume withAddressType(FlashArrayVolume vol) { + if (vol != null) { + vol.setAddressType(volumeAddressType); + } + return vol; } private Object getFlashArrayItem(FlashArrayList list) { @@ -1246,7 +1327,16 @@ public Map getConnectionIdMap(ProviderAdapterDataObject dataIn) if (list != null && list.getItems() != null) { for (FlashArrayConnection conn : list.getItems()) { - if (conn.getHost() != null) { + if (AddressType.NVMETCP.equals(volumeAddressType)) { + // Key on the host name so connid. is matched by + // parseAndValidatePath. NVMe-TCP reports an nsid where Fibre + // Channel reports a lun; the connection itself is host-scoped + // on both transports. + if (conn.getHost() != null && conn.getHost().getName() != null) { + String id = conn.getNsid() != null ? "" + conn.getNsid() : "1"; + map.put(conn.getHost().getName(), id); + } + } else if (conn.getHost() != null) { map.put(conn.getHost().getName(), "" + conn.getLun()); } } diff --git a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayConnection.java b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayConnection.java index 76cec9f70c4b..b115035fda21 100644 --- a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayConnection.java +++ b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayConnection.java @@ -31,6 +31,8 @@ public class FlashArrayConnection { private FlashArrayVolume volume; @JsonProperty("lun") private Integer lun; + @JsonProperty("nsid") + private Integer nsid; public FlashArrayConnectionHostgroup getHostGroup() { return hostGroup; @@ -64,5 +66,12 @@ public void setLun(Integer lun) { this.lun = lun; } + public Integer getNsid() { + return nsid; + } + + public void setNsid(Integer nsid) { + this.nsid = nsid; + } } diff --git a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java index a3201a753a75..8ae232cbc687 100644 --- a/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java +++ b/plugins/storage/volume/flasharray/src/main/java/org/apache/cloudstack/storage/datastore/adapter/flasharray/FlashArrayVolume.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.storage.datastore.adapter.flasharray; +import java.util.regex.Pattern; + import org.apache.cloudstack.storage.datastore.adapter.ProviderSnapshot; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -27,6 +29,13 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class FlashArrayVolume implements ProviderSnapshot { public static final String PURE_OUI = "24a9370"; + // The 3-byte OUI as it appears inside an NVMe EUI-128 (no trailing nibble). + // FC WWNs use a 7-hex-digit Pure OUI; NVMe NGUIDs embed the same vendor + // prefix in its raw 6-hex-digit form. + public static final String PURE_OUI_EUI = "24a937"; + + /** FlashArray volume serials are exactly 24 hexadecimal characters. */ + private static final Pattern SERIAL_PATTERN = Pattern.compile("[0-9a-fA-F]{24}"); @JsonProperty("destroyed") private Boolean destroyed; @@ -107,6 +116,23 @@ public AddressType getAddressType() { @JsonIgnore public String getAddress() { if (serial == null) return null; + if (AddressType.NVMETCP.equals(addressType)) { + // EUI-128 layout for FlashArray NVMe namespaces: + // 00 + serial[0:14] + + serial[14:24] + // This is the value the Linux kernel exposes as + // /dev/disk/by-id/nvme-eui. + // Require an exact 24-hex-character serial. Accepting anything longer and slicing + // the first 24 characters would silently map two distinct serials sharing a prefix + // onto the same EUI, breaking volume identity; accepting anything shorter cannot + // produce a valid 32-character EUI at all. + if (!SERIAL_PATTERN.matcher(serial).matches()) { + throw new RuntimeException("FlashArray serial [" + serial + + "] cannot be used to build an NVMe EUI-128 address " + + "(expected exactly 24 hexadecimal characters, got " + + serial.length() + ")"); + } + return ("00" + serial.substring(0, 14) + PURE_OUI_EUI + serial.substring(14, 24)).toLowerCase(); + } return ("6" + PURE_OUI + serial).toLowerCase(); } @Override diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 934600eb2b61..ee38f6c727c5 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -1355,7 +1355,8 @@ public static HypervisorType getHypervisorTypeFromFormat(long dcId, ImageFormat StoragePoolType.PowerFlex, StoragePoolType.CLVM, StoragePoolType.Linstor, - StoragePoolType.FiberChannel).contains(pool.getPoolType())) { + StoragePoolType.FiberChannel, + StoragePoolType.NVMeTCP).contains(pool.getPoolType())) { // This case will note the presence of non-qcow2 primary stores, suggesting KVM without NFS. Otherwse, // If this check is not passed, the hypervisor type will remain OVM. type = HypervisorType.KVM; diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 8674014addd7..e81cc00564e3 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -2601,7 +2601,8 @@ private void validateVolumeResizeWithSize(VolumeVO volume, long currentSize, Lon if (storagePoolVO.isManaged() && !List.of( Storage.StoragePoolType.PowerFlex, - Storage.StoragePoolType.FiberChannel).contains(storagePoolVO.getPoolType())) { + Storage.StoragePoolType.FiberChannel, + Storage.StoragePoolType.NVMeTCP).contains(storagePoolVO.getPoolType())) { Long instanceId = volume.getInstanceId(); if (instanceId != null) { diff --git a/ui/src/views/infra/AddPrimaryStorage.vue b/ui/src/views/infra/AddPrimaryStorage.vue index 7d189032f098..eeb3ebdd22c6 100644 --- a/ui/src/views/infra/AddPrimaryStorage.vue +++ b/ui/src/views/infra/AddPrimaryStorage.vue @@ -242,7 +242,7 @@ -
+