kvm: encrypt the KVM live-migration data stream with QEMU-native TLS (VIR_MIGRATE_TLS)

PendingReleaseNotes

@@ -39,3 +39,22 @@ example.ver.1 > example.ver.2:
     which can now be attached to Instances. This is to prevent the Secondary
     Storage to grow to enormous sizes as Linux Distributions keep growing in
     size while a stripped down Linux should fit on a 2.88MB floppy.
+
+
+New features:
+
+  * Encrypted KVM live-migration data stream (QEMU-native TLS). CloudStack's
+    secure KVM live migration previously encrypted only the libvirt control
+    channel, leaving the guest memory (and, for storage migration, the disk)
+    stream in plaintext on the migration network. A new zone-scoped global
+    setting 'kvm.migrate.tls' (default false) makes CloudStack request
+    VIR_MIGRATE_TLS so the migration data stream is encrypted and mutually
+    authenticated, reusing the certificates already provisioned by the CA
+    framework (a new /etc/pki/qemu cert set is created by the agent alongside
+    the existing libvirt and VNC ones). TLS is used only when both the source
+    and destination hosts are secured and advertise the 'host.migrate.tls'
+    capability; otherwise the migration transparently falls back to the
+    plaintext stream, so mixed/partially-upgraded fleets keep migrating. To
+    enable it, upgrade the agents fleet-wide, re-provision host certificates
+    (addHost does this automatically; existing hosts via "Deploy Host Keys"),
+    then set 'kvm.migrate.tls=true' for the zone.

api/src/main/java/com/cloud/host/Host.java

@@ -64,6 +64,7 @@ public static String[] toStrings(Host.Type... types) {
     String HOST_VIRTV2V_VERSION = "host.virtv2v.version";
     String HOST_SSH_PORT = "host.ssh.port";
     String HOST_CDROM_MAX_COUNT = "host.cdrom.max.count";
+    String HOST_MIGRATE_TLS = "host.migrate.tls";
     String GUEST_OS_CATEGORY_ID = "guest.os.category.id";
     String GUEST_OS_RULE = "guest.os.rule";
 

core/src/main/java/com/cloud/agent/api/MigrateCommand.java

@@ -44,6 +44,7 @@ public class MigrateCommand extends Command {
 
     private int newVmCpuShares;
     private boolean clvmCrossPoolMigration;
+    private boolean migrateTls;
 
     Map<String, Boolean> vlanToPersistenceMap = new HashMap<>();
 
@@ -159,6 +160,14 @@ public void setClvmCrossPoolMigration(boolean clvmCrossPoolMigration) {
         this.clvmCrossPoolMigration = clvmCrossPoolMigration;
     }
 
+    public boolean isMigrateTls() {
+        return migrateTls;
+    }
+
+    public void setMigrateTls(boolean migrateTls) {
+        this.migrateTls = migrateTls;
+    }
+
     public static class MigrateDiskInfo {
         public enum DiskType {
             FILE, BLOCK;

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -107,6 +107,16 @@ public interface StorageManager extends StorageService {
             true,
             ConfigKey.Scope.Global,
             null);
+    ConfigKey<Boolean> KvmMigrateTls = new ConfigKey<>(Boolean.class,
+            "kvm.migrate.tls",
+            "Storage",
+            "false",
+            "Setting this to 'true' encrypts the KVM live-migration data stream (guest memory and non-shared disk) with QEMU-native TLS (VIR_MIGRATE_TLS), " +
+            "provided both the source and destination hosts are secured and advertise host.migrate.tls support. If either host does not support it, the migration " +
+            "silently falls back to the plaintext data stream. Reuses the certificates provisioned by the CA framework.",
+            true,
+            ConfigKey.Scope.Zone,
+            null);
     ConfigKey<Integer> MaxNumberOfManagedClusteredFileSystems = new ConfigKey<>(Integer.class,
             "max.number.managed.clustered.file.systems",
             "Storage",

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -217,6 +217,7 @@
 import com.cloud.exception.StorageUnavailableException;
 import com.cloud.ha.HighAvailabilityManager;
 import com.cloud.ha.HighAvailabilityManager.WorkType;
+import com.cloud.host.DetailVO;
 import com.cloud.host.Host;
 import com.cloud.host.HostVO;
 import com.cloud.host.Status;
@@ -3383,6 +3384,8 @@ protected MigrateCommand buildMigrateCommand(VMInstanceVO vmInstance, VirtualMac
         migrateCommand.setAutoConvergence(StorageManager.KvmAutoConvergence.value());
         migrateCommand.setHostGuid(destination.getHost().getGuid());
 
+        migrateCommand.setMigrateTls(shouldMigrateWithTls(vmInstance, destination));
+
         PrepareForMigrationAnswer prepareForMigrationAnswer = (PrepareForMigrationAnswer) answer;
 
         Map<String, DpdkTO> answerDpdkInterfaceMapping = prepareForMigrationAnswer.getDpdkInterfaceMapping();
@@ -3402,6 +3405,40 @@ protected MigrateCommand buildMigrateCommand(VMInstanceVO vmInstance, VirtualMac
         return migrateCommand;
     }
 
+    /**
+     * Decides whether the KVM live-migration data stream should be encrypted with QEMU-native TLS.
+     * TLS is used only when the {@code kvm.migrate.tls} setting is enabled for the destination zone
+     * AND both the source and destination hosts advertise {@code host.migrate.tls} support. If any of
+     * these conditions is not met the migration silently falls back to the plaintext {@code tcp:} stream,
+     * which keeps mixed / partially-upgraded fleets migrating without failures.
+     */
+    protected boolean shouldMigrateWithTls(VMInstanceVO vmInstance, DeployDestination destination) {
+        final Long zoneId = destination.getHost().getDataCenterId();
+        if (!StorageManager.KvmMigrateTls.valueIn(zoneId)) {
+            return false;
+        }
+
+        final Long srcHostId = vmInstance.getHostId() != null ? vmInstance.getHostId() : vmInstance.getLastHostId();
+        final Long destHostId = destination.getHost().getId();
+        final boolean srcSupportsTls = srcHostId != null && hostAdvertisesMigrateTls(srcHostId);
+        final boolean destSupportsTls = destHostId != null && hostAdvertisesMigrateTls(destHostId);
+
+        if (!srcSupportsTls || !destSupportsTls) {
+            logger.debug("kvm.migrate.tls is enabled but not both hosts advertise migration-TLS support (source [{}]={}, destination [{}]={}) for VM [{}]; " +
+                    "falling back to the plaintext migration data stream.", srcHostId, srcSupportsTls, destHostId, destSupportsTls, vmInstance.getInstanceName());
+            return false;
+        }
+
+        logger.debug("Enabling QEMU-native TLS for the migration data stream of VM [{}] (source host [{}], destination host [{}]).",
+                vmInstance.getInstanceName(), srcHostId, destHostId);
+        return true;
+    }
+
+    private boolean hostAdvertisesMigrateTls(Long hostId) {
+        final DetailVO detail = hostDetailsDao.findDetail(hostId, Host.HOST_MIGRATE_TLS);
+        return detail != null && Boolean.parseBoolean(detail.getValue());
+    }
+
     private void updateVmPod(VMInstanceVO vm, long dstHostId) {
         // update the VMs pod
         HostVO host = _hostDao.findById(dstHostId);

engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java

@@ -105,9 +105,11 @@
 import com.cloud.dc.dao.ClusterDao;
 import com.cloud.exception.AgentUnavailableException;
 import com.cloud.exception.OperationTimedoutException;
+import com.cloud.host.DetailVO;
 import com.cloud.host.Host;
 import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
+import com.cloud.host.dao.HostDetailsDao;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.resource.ResourceState;
 import com.cloud.storage.DataStoreRole;
@@ -179,6 +181,8 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy {
     @Inject
     private HostDao _hostDao;
     @Inject
+    private HostDetailsDao _hostDetailsDao;
+    @Inject
     protected PrimaryDataStoreDao _storagePoolDao;
     @Inject
     private SnapshotDao _snapshotDao;
@@ -2202,6 +2206,8 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
             boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value();
             migrateCommand.setAutoConvergence(kvmAutoConvergence);
 
+            migrateCommand.setMigrateTls(shouldMigrateWithTls(srcHost, destHost));
+
             MigrateAnswer migrateAnswer = null;
             try {
                 migrateAnswer = (MigrateAnswer)agentManager.send(srcHost.getId(), migrateCommand);
@@ -2349,6 +2355,36 @@ protected boolean shouldMigrateVolume(StoragePoolVO sourceStoragePool, Host dest
         return true;
     }
 
+    /**
+     * Decides whether the KVM live-migration-with-volumes data stream should be encrypted with QEMU-native TLS.
+     * This path carries both guest memory and disk contents on the wire. TLS is used only when the
+     * {@code kvm.migrate.tls} setting is enabled for the destination zone AND both the source and destination
+     * hosts advertise {@code host.migrate.tls} support; otherwise it silently falls back to the plaintext stream,
+     * keeping mixed / partially-upgraded fleets migrating without failures.
+     */
+    protected boolean shouldMigrateWithTls(Host srcHost, Host destHost) {
+        if (srcHost == null || destHost == null) {
+            return false;
+        }
+        if (!StorageManager.KvmMigrateTls.valueIn(destHost.getDataCenterId())) {
+            return false;
+        }
+
+        final boolean srcSupportsTls = hostAdvertisesMigrateTls(srcHost.getId());
+        final boolean destSupportsTls = hostAdvertisesMigrateTls(destHost.getId());
+        if (!srcSupportsTls || !destSupportsTls) {
+            logger.debug("kvm.migrate.tls is enabled but not both hosts advertise migration-TLS support (source [{}]={}, destination [{}]={}); " +
+                    "falling back to the plaintext migration data stream.", srcHost.getId(), srcSupportsTls, destHost.getId(), destSupportsTls);
+            return false;
+        }
+        return true;
+    }
+
+    private boolean hostAdvertisesMigrateTls(Long hostId) {
+        final DetailVO detail = _hostDetailsDao.findDetail(hostId, Host.HOST_MIGRATE_TLS);
+        return detail != null && Boolean.parseBoolean(detail.getValue());
+    }
+
     /**
      * Returns true if the storage pool type is {@link StoragePoolType.Filesystem}.
      */

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -18,6 +18,7 @@
 
 import static com.cloud.host.Host.HOST_CDROM_MAX_COUNT;
 import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION;
+import static com.cloud.host.Host.HOST_MIGRATE_TLS;
 import static com.cloud.host.Host.HOST_OVFTOOL_VERSION;
 import static com.cloud.host.Host.HOST_VDDK_LIB_DIR;
 import static com.cloud.host.Host.HOST_VDDK_SUPPORT;
@@ -4410,6 +4411,7 @@ public StartupCommand[] initialize() {
         cmd.setGatewayIpAddress(localGateway);
         cmd.setIqn(getIqn());
         cmd.getHostDetails().put(HOST_VOLUME_ENCRYPTION, String.valueOf(hostSupportsVolumeEncryption()));
+        cmd.getHostDetails().put(HOST_MIGRATE_TLS, String.valueOf(hostSupportsMigrateTls()));
         cmd.setHostTags(getHostTags());
         boolean instanceConversionSupported = hostSupportsInstanceConversion();
         cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(instanceConversionSupported));
@@ -6220,6 +6222,42 @@ public boolean hostSupportsVolumeEncryption() {
         return true;
     }
 
+    protected static final String QEMU_MIGRATE_TLS_CONF_FILE = "/etc/libvirt/qemu.conf";
+    protected static final String QEMU_MIGRATE_TLS_CERT_DIR = "/etc/pki/qemu";
+
+    /**
+     * Determines whether this host can encrypt the live-migration data stream with QEMU-native TLS.
+     * Both conditions provisioned by the CA framework (keystore-cert-import + configure_libvirt_tls)
+     * must hold: the QEMU migration certificates exist under {@link #QEMU_MIGRATE_TLS_CERT_DIR} and
+     * qemu.conf points migration TLS at that directory via {@code migrate_tls_x509_cert_dir}. The
+     * result is advertised to the management server as the {@code host.migrate.tls} host detail.
+     */
+    public boolean hostSupportsMigrateTls() {
+        final File certDir = new File(QEMU_MIGRATE_TLS_CERT_DIR);
+        final File serverCert = new File(certDir, "server-cert.pem");
+        final File serverKey = new File(certDir, "server-key.pem");
+        final File caCert = new File(certDir, "ca-cert.pem");
+        if (!serverCert.exists() || !serverKey.exists() || !caCert.exists()) {
+            return false;
+        }
+
+        final File qemuConf = new File(QEMU_MIGRATE_TLS_CONF_FILE);
+        if (!qemuConf.exists()) {
+            return false;
+        }
+        try {
+            for (final String line : Files.readAllLines(qemuConf.toPath())) {
+                final String normalized = line.trim();
+                if (!normalized.startsWith("#") && normalized.replaceAll("\\s", "").startsWith("migrate_tls_x509_cert_dir=")) {
+                    return true;
+                }
+            }
+        } catch (final IOException e) {
+            LOGGER.warn("Unable to read {} to determine migration-TLS support", QEMU_MIGRATE_TLS_CONF_FILE, e);
+        }
+        return false;
+    }
+
     public boolean isSecureMode(String bootMode) {
         if (StringUtils.isNotBlank(bootMode) && "secure".equalsIgnoreCase(bootMode)) {
             return true;

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/MigrateKVMAsync.java

@@ -44,6 +44,7 @@ public class MigrateKVMAsync implements Callable<Domain> {
     private boolean migrateStorage;
     private boolean migrateNonSharedInc;
     private boolean autoConvergence;
+    private boolean migrateTls;
 
     protected Set<String> migrateDiskLabels;
 
@@ -90,14 +91,22 @@ public class MigrateKVMAsync implements Callable<Domain> {
     // to tune the algorithm.
     private static final long VIR_MIGRATE_AUTO_CONVERGE = 8192L;
 
+    // Use TLS for the native (QEMU) migration data connection. When set, QEMU encrypts
+    // the guest memory (and non-shared disk) stream using the certificates configured
+    // via qemu.conf's migrate_tls_x509_cert_dir. TLS is negotiated over the normal
+    // "tcp:" data connection - libvirt has no "tls:" migration URI scheme (valid
+    // schemes are tcp/rdma/unix/fd), so the URI is unchanged and only this flag is set.
+    private static final long VIR_MIGRATE_TLS = 65536L;
+
     // Libvirt 1.0.3 supports compression flag for migration.
     private static final int LIBVIRT_VERSION_SUPPORTS_MIGRATE_COMPRESSED = 1000003;
 
     // Libvirt 1.2.3 supports auto converge.
     private static final int LIBVIRT_VERSION_SUPPORTS_AUTO_CONVERGE = 1002003;
 
     public MigrateKVMAsync(final LibvirtComputingResource libvirtComputingResource, final Domain dm, final Connect dconn, final String dxml,
-            final boolean migrateStorage, final boolean migrateNonSharedInc, final boolean autoConvergence, final String vmName, final String destIp, Set<String> migrateDiskLabels) {
+            final boolean migrateStorage, final boolean migrateNonSharedInc, final boolean autoConvergence, final String vmName, final String destIp, Set<String> migrateDiskLabels,
+            final boolean migrateTls) {
         this.libvirtComputingResource = libvirtComputingResource;
 
         this.dm = dm;
@@ -109,6 +118,7 @@ public MigrateKVMAsync(final LibvirtComputingResource libvirtComputingResource,
         this.vmName = vmName;
         this.destIp = destIp;
         this.migrateDiskLabels = migrateDiskLabels;
+        this.migrateTls = migrateTls;
     }
 
     @Override
@@ -134,6 +144,11 @@ public Domain call() throws LibvirtException {
             flags |= VIR_MIGRATE_AUTO_CONVERGE;
         }
 
+        if (migrateTls) {
+            flags |= VIR_MIGRATE_TLS;
+            logger.debug("Setting VIR_MIGRATE_TLS to encrypt the migration data stream of {}.", vmName);
+        }
+
         TypedParameter [] parameters = createTypedParameterList();
 
         logger.debug(String.format("Migrating [%s] with flags [%s], destination [%s] and speed [%s]. The disks with the following labels will be migrated [%s].", vmName, flags,

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java

@@ -252,7 +252,7 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0.
 
             final Callable<Domain> worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc,
                     migrateStorage, migrateNonSharedInc,
-                    command.isAutoConvergence(), vmName, command.getDestinationIp(), migrateDiskLabels);
+                    command.isAutoConvergence(), vmName, command.getDestinationIp(), migrateDiskLabels, command.isMigrateTls());
             final Future<Domain> migrateThread = executor.submit(worker);
             executor.shutdown();
             long sleeptime = 0;

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/MigrateKVMAsyncTest.java

@@ -45,7 +45,7 @@ public class MigrateKVMAsyncTest {
     @Test
     public void createTypedParameterListTestNoMigrateDiskLabels() {
         MigrateKVMAsync migrateKVMAsync = new MigrateKVMAsync(libvirtComputingResource, domain, connect, "testxml",
-                false, false, false, "tst", "1.1.1.1", null);
+                false, false, false, "tst", "1.1.1.1", null, false);
 
         Mockito.doReturn(10).when(libvirtComputingResource).getMigrateSpeed();
 
@@ -60,11 +60,30 @@ public void createTypedParameterListTestNoMigrateDiskLabels() {
 
     }
 
+    @Test
+    public void createTypedParameterListTestWithMigrateTlsKeepsTcpUri() {
+        // TLS is enabled solely via the VIR_MIGRATE_TLS flag; the data URI must stay "tcp:"
+        // because libvirt has no "tls:" migration URI scheme. Guards against reintroducing it.
+        MigrateKVMAsync migrateKVMAsync = new MigrateKVMAsync(libvirtComputingResource, domain, connect, "testxml",
+                false, false, false, "tst", "1.1.1.1", null, true);
+
+        Mockito.doReturn(10).when(libvirtComputingResource).getMigrateSpeed();
+
+        TypedParameter[] result = migrateKVMAsync.createTypedParameterList();
+
+        Assert.assertEquals(4, result.length);
+
+        Assert.assertEquals("tst", result[0].getValueAsString());
+        Assert.assertEquals("testxml", result[1].getValueAsString());
+        Assert.assertEquals("tcp:1.1.1.1", result[2].getValueAsString());
+        Assert.assertEquals("10", result[3].getValueAsString());
+    }
+
     @Test
     public void createTypedParameterListTestWithMigrateDiskLabels() {
         Set<String> labels = Set.of("vda", "vdb");
         MigrateKVMAsync migrateKVMAsync = new MigrateKVMAsync(libvirtComputingResource, domain, connect, "testxml",
-                false, false, false, "tst", "1.1.1.1", labels);
+                false, false, false, "tst", "1.1.1.1", labels, false);
 
         Mockito.doReturn(10).when(libvirtComputingResource).getMigrateSpeed();
 

python/lib/cloudutils/serviceConfig.py

@@ -598,6 +598,10 @@ def configure_libvirt_tls(tls_enabled=False, cfo=None):
         cfo.addEntry("vnc_tls", "1")
         cfo.addEntry("vnc_tls_x509_verify", "1")
         cfo.addEntry("vnc_tls_x509_cert_dir", "\"/etc/pki/libvirt-vnc\"")
+        # QEMU native-TLS for the live-migration data stream (VIR_MIGRATE_TLS).
+        # Reuses the CA-framework certificates provisioned under /etc/pki/qemu.
+        cfo.addEntry("migrate_tls_x509_cert_dir", "\"/etc/pki/qemu\"")
+        cfo.addEntry("migrate_tls_x509_verify", "1")
     else:
         cfo.addEntry("vnc_tls", "0")