flasharray/kvm/adaptive: NVMe-TCP transport for FlashArray primary storage

PendingReleaseNotes

@@ -39,3 +39,15 @@ example.ver.1 > example.ver.2:
     which can now be attached to Instances. This is to prevent the Secondary
     Storage to grow to enormous sizes as Linux Distributions keep growing in
     size while a stripped down Linux should fit on a 2.88MB floppy.
+
+
+4.22.0 > 4.23.0:
+  * Added NVMe-over-Fabrics (TCP) support to the adaptive storage framework
+    and the Pure Storage FlashArray plugin. Volumes on a FlashArray primary
+    pool can now be delivered to KVM hypervisors over NVMe-TCP instead of
+    Fibre Channel by setting transport=nvme-tcp on the pool's provider URL.
+    Volumes are identified on the host via EUI-128 NGUIDs and attached to
+    guests as plain block devices through the native NVMe multipath layer;
+    no device-mapper multipath configuration is required. A new
+    Storage.StoragePoolType.NVMeTCP + MultipathNVMeOFAdapterBase /
+    NVMeTCPAdapter on the KVM side back the new pool type.

api/src/main/java/com/cloud/storage/Storage.java

@@ -185,7 +185,8 @@ public static enum StoragePoolType {
         Linstor(true, true, EncryptionSupport.Storage),
         DatastoreCluster(true, true, EncryptionSupport.Unsupported), // for VMware, to abstract pool of clusters
         StorPool(true, true, EncryptionSupport.Hypervisor),
-        FiberChannel(true, true, EncryptionSupport.Unsupported); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-<wwnvalue>)
+        FiberChannel(true, true, EncryptionSupport.Unsupported), // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-<wwnvalue>)
+        NVMeTCP(true, true, EncryptionSupport.Unsupported); // NVMe over TCP (NVMe-oF/TCP) Pool for KVM hypervisors; volumes are identified by EUI-128 NGUID (/dev/disk/by-id/nvme-eui.<eui>)
 
         private final boolean shared;
         private final boolean overProvisioning;

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -415,7 +415,8 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) {
                     StoragePoolType.PowerFlex,
                     StoragePoolType.Linstor,
                     StoragePoolType.FiberChannel,
-                    StoragePoolType.CLVM).contains(primaryPool.getType())) {
+                    StoragePoolType.CLVM,
+                    StoragePoolType.NVMeTCP).contains(primaryPool.getType())) {
                     newTemplate.setFormat(ImageFormat.RAW);
                 } else {
                     newTemplate.setFormat(ImageFormat.QCOW2);
@@ -448,7 +449,8 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) {
 
     public static String derivePath(PrimaryDataStoreTO primaryStore, DataTO destData, Map<String, String> details) {
         String path = null;
-        if (primaryStore.getPoolType() == StoragePoolType.FiberChannel) {
+        if (primaryStore.getPoolType() == StoragePoolType.FiberChannel
+                || primaryStore.getPoolType() == StoragePoolType.NVMeTCP) {
             path = destData.getPath();
         } else {
             path = details != null ? details.get("managedStoreTarget") : null;
@@ -3431,7 +3433,8 @@ private Storage.ImageFormat getFormat(StoragePoolType poolType) {
                 StoragePoolType.PowerFlex,
                 StoragePoolType.Linstor,
                 StoragePoolType.FiberChannel,
-                StoragePoolType.CLVM).contains(poolType)) {
+                StoragePoolType.CLVM,
+                StoragePoolType.NVMeTCP).contains(poolType)) {
             return ImageFormat.RAW;
         } else {
             return ImageFormat.QCOW2;

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathNVMeOFPool.java

@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.kvm.storage;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cloudstack.utils.qemu.QemuImg;
+import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
+import org.joda.time.Duration;
+
+import com.cloud.agent.api.to.HostTO;
+import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool;
+import com.cloud.storage.Storage;
+import com.cloud.storage.Storage.ProvisioningType;
+
+/**
+ * KVMStoragePool for NVMe-over-Fabrics pools. Mirror of
+ * {@link MultipathSCSIPool} for adapters based on
+ * {@link MultipathNVMeOFAdapterBase}. Every data operation is delegated
+ * back to the adapter; the pool itself only tracks addressing/identity.
+ */
+public class MultipathNVMeOFPool implements KVMStoragePool {
+    private final String uuid;
+    private final String sourceHost;
+    private final int sourcePort;
+    private final String sourceDir;
+    private final Storage.StoragePoolType storagePoolType;
+    private final StorageAdaptor storageAdaptor;
+    private final Map<String, String> details;
+    private long capacity;
+    private long used;
+    private long available;
+
+    public MultipathNVMeOFPool(String uuid, String host, int port, String path,
+            Storage.StoragePoolType poolType, Map<String, String> poolDetails, StorageAdaptor adaptor) {
+        this.uuid = uuid;
+        this.sourceHost = host;
+        this.sourcePort = port;
+        this.sourceDir = path;
+        this.storagePoolType = poolType;
+        this.storageAdaptor = adaptor;
+        this.details = poolDetails;
+        this.capacity = 0;
+        this.used = 0;
+        this.available = 0;
+    }
+
+    public MultipathNVMeOFPool(String uuid, StorageAdaptor adaptor) {
+        this.uuid = uuid;
+        this.sourceHost = null;
+        this.sourcePort = -1;
+        this.sourceDir = null;
+        this.storagePoolType = Storage.StoragePoolType.NVMeTCP;
+        this.storageAdaptor = adaptor;
+        this.details = new HashMap<>();
+        this.capacity = 0;
+        this.used = 0;
+        this.available = 0;
+    }
+
+    @Override
+    public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, ProvisioningType provisioningType, long size, byte[] passphrase) {
+        return null;
+    }
+
+    @Override
+    public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, ProvisioningType provisioningType, long size, byte[] passphrase) {
+        return null;
+    }
+
+    @Override
+    public boolean connectPhysicalDisk(String volumeUuid, Map<String, String> details) {
+        return storageAdaptor.connectPhysicalDisk(volumeUuid, this, details, false);
+    }
+
+    @Override
+    public KVMPhysicalDisk getPhysicalDisk(String volumeId) {
+        return storageAdaptor.getPhysicalDisk(volumeId, this);
+    }
+
+    @Override
+    public boolean disconnectPhysicalDisk(String volumeUuid) {
+        return storageAdaptor.disconnectPhysicalDisk(volumeUuid, this);
+    }
+
+    @Override
+    public boolean deletePhysicalDisk(String volumeUuid, Storage.ImageFormat format) {
+        return true;
+    }
+
+    @Override
+    public List<KVMPhysicalDisk> listPhysicalDisks() {
+        return null;
+    }
+
+    @Override
+    public String getUuid() {
+        return uuid;
+    }
+
+    public void setCapacity(long capacity) { this.capacity = capacity; }
+    @Override public long getCapacity() { return this.capacity; }
+    public void setUsed(long used) { this.used = used; }
+    @Override public long getUsed() { return this.used; }
+    public void setAvailable(long available) { this.available = available; }
+    @Override public long getAvailable() { return this.available; }
+
+    @Override public boolean refresh() { return false; }
+    @Override public boolean isExternalSnapshot() { return true; }
+    @Override public String getLocalPath() { return null; }
+    @Override public String getSourceHost() { return this.sourceHost; }
+    @Override public String getSourceDir() { return this.sourceDir; }
+    @Override public int getSourcePort() { return this.sourcePort; }
+    @Override public String getAuthUserName() { return null; }
+    @Override public String getAuthSecret() { return null; }
+    @Override public Storage.StoragePoolType getType() { return storagePoolType; }
+    @Override public boolean delete() { return false; }
+    @Override public QemuImg.PhysicalDiskFormat getDefaultFormat() { return QemuImg.PhysicalDiskFormat.RAW; }
+    @Override public boolean createFolder(String path) { return false; }
+    @Override public boolean supportsConfigDriveIso() { return false; }
+    @Override public Map<String, String> getDetails() { return this.details; }
+    @Override public boolean isPoolSupportHA() { return false; }
+    @Override public String getHearthBeatPath() { return null; }
+
+    @Override
+    public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) {
+        return null;
+    }
+
+    @Override public String getStorageNodeId() { return null; }
+
+    @Override
+    public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { return null; }
+
+    @Override
+    public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout,
+            String volumeUUIDListString, String vmActivityCheckPath, long duration) {
+        return null;
+    }
+}

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/NVMeTCPAdapter.java

@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.kvm.storage;
+
+import com.cloud.storage.Storage;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * StorageAdaptor for the {@link Storage.StoragePoolType#NVMeTCP} pool type.
+ * All operational logic lives in {@link MultipathNVMeOFAdapterBase}; this
+ * class just binds that logic to a pool type so
+ * {@link KVMStoragePoolManager} can find it via reflection.
+ */
+public class NVMeTCPAdapter extends MultipathNVMeOFAdapterBase {
+    private static final Logger LOGGER = LogManager.getLogger(NVMeTCPAdapter.class);
+
+    public NVMeTCPAdapter() {
+        LOGGER.info("Loaded NVMeTCPAdapter for StorageLayer");
+    }
+
+    @Override
+    public String getName() {
+        return "NVMeTCPAdapter";
+    }
+
+    @Override
+    public Storage.StoragePoolType getStoragePoolType() {
+        return Storage.StoragePoolType.NVMeTCP;
+    }
+
+    @Override
+    public boolean isStoragePoolTypeSupported(Storage.StoragePoolType type) {
+        return Storage.StoragePoolType.NVMeTCP.equals(type);
+    }
+}

plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderVolume.java

@@ -35,6 +35,7 @@ public interface ProviderVolume {
     public String getExternalName();
     public String getExternalConnectionId();
     public enum AddressType {
-        FIBERWWN
+        FIBERWWN,
+        NVMETCP
     }
 }

plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/AdaptiveDataStoreLifeCycleImpl.java

@@ -179,7 +179,7 @@ public DataStore initialize(Map<String, Object> dsInfos) {
             parameters.setHost(uri.getHost());
             parameters.setPort(uri.getPort());
             parameters.setPath(uri.getPath() + "?" + uri.getQuery());
-            parameters.setType(StoragePoolType.FiberChannel);
+            parameters.setType(pickPoolType(uri));
             parameters.setZoneId(zoneId);
             parameters.setPodId(podId);
             parameters.setClusterId(clusterId);
@@ -402,4 +402,26 @@ public void disableStoragePool(DataStore store) {
         logger.info("Disabling storage pool {}", store);
         _dataStoreHelper.disable(store);
     }
+
+    /**
+     * Resolve the CloudStack StoragePoolType from the provider URL. Adaptive
+     * plugins advertise the underlying fabric via a {@code transport=} query
+     * parameter on the URL; when absent we keep the legacy FiberChannel
+     * default for backwards compatibility with adapters that still assume it.
+     */
+    private static StoragePoolType pickPoolType(java.net.URL uri) {
+        String query = uri.getQuery();
+        if (query != null) {
+            for (String tok : query.split("&")) {
+                int i = tok.indexOf('=');
+                if (i > 0 && "transport".equalsIgnoreCase(tok.substring(0, i))) {
+                    String value = tok.substring(i + 1);
+                    if ("nvme-tcp".equalsIgnoreCase(value)) {
+                        return StoragePoolType.NVMeTCP;
+                    }
+                }
+            }
+        }
+        return StoragePoolType.FiberChannel;
+    }
 }