diff --git a/base/cvd/cuttlefish/host/commands/assemble_cvd/assemble_cvd_flags.cpp b/base/cvd/cuttlefish/host/commands/assemble_cvd/assemble_cvd_flags.cpp index 96778f9bdec..d6bdc7681f0 100644 --- a/base/cvd/cuttlefish/host/commands/assemble_cvd/assemble_cvd_flags.cpp +++ b/base/cvd/cuttlefish/host/commands/assemble_cvd/assemble_cvd_flags.cpp @@ -339,6 +339,11 @@ DEFINE_vec(protected_vm, fmt::format("{}", CF_DEFAULTS_PROTECTED_VM), DEFINE_vec(mte, fmt::format("{}", CF_DEFAULTS_MTE), "Enable MTE"); +DEFINE_vec(enable_pkvm, fmt::format("{}", CF_DEFAULTS_ENABLE_PKVM), + "Provision the guest to run pKVM so it can host its own protected " + "VMs; requires a nested-virt capable host, --vm_manager=crosvm " + "and an arm64 guest."); + DEFINE_vec(enable_audio, fmt::format("{}", CF_DEFAULTS_ENABLE_AUDIO), "Whether to play or capture audio"); diff --git a/base/cvd/cuttlefish/host/commands/assemble_cvd/assemble_cvd_flags.h b/base/cvd/cuttlefish/host/commands/assemble_cvd/assemble_cvd_flags.h index d0501b7fde3..53419829155 100644 --- a/base/cvd/cuttlefish/host/commands/assemble_cvd/assemble_cvd_flags.h +++ b/base/cvd/cuttlefish/host/commands/assemble_cvd/assemble_cvd_flags.h @@ -160,6 +160,8 @@ DECLARE_vec(protected_vm); DECLARE_vec(mte); +DECLARE_vec(enable_pkvm); + DECLARE_vec(enable_audio); DECLARE_vec(enable_jcard_simulator); diff --git a/base/cvd/cuttlefish/host/commands/assemble_cvd/bootconfig_args.cpp b/base/cvd/cuttlefish/host/commands/assemble_cvd/bootconfig_args.cpp index 6d6c33fe99c..8b191536390 100644 --- a/base/cvd/cuttlefish/host/commands/assemble_cvd/bootconfig_args.cpp +++ b/base/cvd/cuttlefish/host/commands/assemble_cvd/bootconfig_args.cpp @@ -263,15 +263,23 @@ Result> BootconfigArgsFromConfig( } // TODO(b/217564326): improve this checks for a hypervisor in the VM. - if (instance.target_arch() == Arch::X86 || - instance.target_arch() == Arch::X86_64) { - bootconfig_args["androidboot.hypervisor.version"] = - "cf-" + ToString(config.vm_manager()); - bootconfig_args["androidboot.hypervisor.vm.supported"] = "1"; - } else { - bootconfig_args["androidboot.hypervisor.vm.supported"] = "0"; + switch (instance.target_arch()) { + case Arch::Arm64: + // The guest bootloader reports the hypervisor properties. + break; + case Arch::X86: + case Arch::X86_64: + bootconfig_args["androidboot.hypervisor.version"] = + "cf-" + ToString(config.vm_manager()); + bootconfig_args["androidboot.hypervisor.vm.supported"] = "1"; + bootconfig_args["androidboot.hypervisor.protected_vm.supported"] = "0"; + break; + case Arch::Arm: + case Arch::RiscV64: + bootconfig_args["androidboot.hypervisor.vm.supported"] = "0"; + bootconfig_args["androidboot.hypervisor.protected_vm.supported"] = "0"; + break; } - bootconfig_args["androidboot.hypervisor.protected_vm.supported"] = "0"; if (!instance.kernel_path().empty()) { bootconfig_args["androidboot.kernel_hotswapped"] = "1"; } diff --git a/base/cvd/cuttlefish/host/commands/assemble_cvd/disk/android_composite_disk_config.cc b/base/cvd/cuttlefish/host/commands/assemble_cvd/disk/android_composite_disk_config.cc index 2c31ea21708..f5743b2e9d3 100644 --- a/base/cvd/cuttlefish/host/commands/assemble_cvd/disk/android_composite_disk_config.cc +++ b/base/cvd/cuttlefish/host/commands/assemble_cvd/disk/android_composite_disk_config.cc @@ -50,6 +50,7 @@ constexpr struct { std::string_view init_boot = "init_boot"; std::string_view metadata = "metadata"; std::string_view misc = "misc"; + std::string_view pvmfw = "pvmfw"; std::string_view super = "super"; std::string_view userdata = "userdata"; std::string_view vbmeta = "vbmeta"; @@ -83,6 +84,7 @@ Result> AndroidCompositeDiskConfig( const std::set ab_partitions = { kPartitions.boot, kPartitions.init_boot, + kPartitions.pvmfw, kPartitions.vbmeta, kPartitions.vbmeta_system, kPartitions.vbmeta_system_dlkm, diff --git a/base/cvd/cuttlefish/host/commands/assemble_cvd/flags.cc b/base/cvd/cuttlefish/host/commands/assemble_cvd/flags.cc index e844633867a..f1bba2628fb 100644 --- a/base/cvd/cuttlefish/host/commands/assemble_cvd/flags.cc +++ b/base/cvd/cuttlefish/host/commands/assemble_cvd/flags.cc @@ -580,6 +580,8 @@ Result InitializeCuttlefishConfiguration( std::vector vhost_net_vec = CF_EXPECT(GET_FLAG_BOOL_VALUE(vhost_net)); std::vector vhost_user_vsock_vec = CF_EXPECT(GET_FLAG_STR_VALUE(vhost_user_vsock)); + std::vector enable_pkvm_vec = + CF_EXPECT(GET_FLAG_BOOL_VALUE(enable_pkvm)); std::vector ril_dns_vec = CF_EXPECT(GET_FLAG_STR_VALUE(ril_dns)); std::vector enable_jcard_simulator_vec = CF_EXPECT(GET_FLAG_BOOL_VALUE(enable_jcard_simulator)); @@ -907,6 +909,14 @@ Result InitializeCuttlefishConfiguration( vhost_user_vsock_vec[instance_index]); } + if (enable_pkvm_vec[instance_index]) { + CF_EXPECT_EQ(tmp_config_obj.vm_manager(), VmmMode::kCrosvm, + "Only crosvm supports --enable_pkvm"); + CF_EXPECT_EQ(guest_configs[instance_index].target_arch, Arch::Arm64, + "--enable_pkvm requires an arm64 guest"); + } + instance.set_enable_pkvm(enable_pkvm_vec[instance_index]); + if (use_random_serial_vec[instance_index]) { instance.set_serial_number( RandomSerialNumber("CFCVD" + std::to_string(num))); diff --git a/base/cvd/cuttlefish/host/commands/assemble_cvd/flags_defaults.h b/base/cvd/cuttlefish/host/commands/assemble_cvd/flags_defaults.h index cb10cfbbb69..1bf2d62bef9 100644 --- a/base/cvd/cuttlefish/host/commands/assemble_cvd/flags_defaults.h +++ b/base/cvd/cuttlefish/host/commands/assemble_cvd/flags_defaults.h @@ -100,6 +100,7 @@ #define CF_DEFAULTS_SECURE_HALS CF_DEFAULTS_DYNAMIC_STRING #define CF_DEFAULTS_PROTECTED_VM false #define CF_DEFAULTS_MTE false +#define CF_DEFAULTS_ENABLE_PKVM false // Kernel default parameters #define CF_DEFAULTS_ENABLE_KERNEL_LOG true diff --git a/base/cvd/cuttlefish/host/commands/assemble_cvd/super_image_mixer.cc b/base/cvd/cuttlefish/host/commands/assemble_cvd/super_image_mixer.cc index d53a21ddd08..359a1134757 100644 --- a/base/cvd/cuttlefish/host/commands/assemble_cvd/super_image_mixer.cc +++ b/base/cvd/cuttlefish/host/commands/assemble_cvd/super_image_mixer.cc @@ -48,14 +48,23 @@ namespace { constexpr char kMiscInfoPath[] = "META/misc_info.txt"; constexpr char kDynamicPartitionsPath[] = "META/dynamic_partitions_info.txt"; constexpr std::array kVendorTargetImages = { - "IMAGES/boot.img", "IMAGES/dtbo.img", - "IMAGES/init_boot.img", "IMAGES/odm.img", - "IMAGES/odm_dlkm.img", "IMAGES/recovery.img", - "IMAGES/system_dlkm.img", "IMAGES/userdata.img", - "IMAGES/vbmeta.img", "IMAGES/vbmeta_system_dlkm.img", - "IMAGES/vbmeta_vendor.img", "IMAGES/vbmeta_vendor_dlkm.img", - "IMAGES/vendor.img", "IMAGES/vendor_boot.img", - "IMAGES/vendor_dlkm.img", "IMAGES/vendor_kernel_boot.img", + "IMAGES/boot.img", + "IMAGES/dtbo.img", + "IMAGES/init_boot.img", + "IMAGES/odm.img", + "IMAGES/odm_dlkm.img", + "IMAGES/pvmfw.img", + "IMAGES/recovery.img", + "IMAGES/system_dlkm.img", + "IMAGES/userdata.img", + "IMAGES/vbmeta.img", + "IMAGES/vbmeta_system_dlkm.img", + "IMAGES/vbmeta_vendor.img", + "IMAGES/vbmeta_vendor_dlkm.img", + "IMAGES/vendor.img", + "IMAGES/vendor_boot.img", + "IMAGES/vendor_dlkm.img", + "IMAGES/vendor_kernel_boot.img", }; constexpr std::array kVendorTargetBuildProps = { "ODM/build.prop", diff --git a/base/cvd/cuttlefish/host/commands/assemble_cvd/vendor_dlkm_utils.cc b/base/cvd/cuttlefish/host/commands/assemble_cvd/vendor_dlkm_utils.cc index 5706837a937..ccc5216029a 100644 --- a/base/cvd/cuttlefish/host/commands/assemble_cvd/vendor_dlkm_utils.cc +++ b/base/cvd/cuttlefish/host/commands/assemble_cvd/vendor_dlkm_utils.cc @@ -131,6 +131,7 @@ std::vector GetRamdiskModules( "vmw_vsock_virtio_transport.ko", "vmw_vsock_virtio_transport_common.ko", "vsock.ko", + "pkvm_iommu_temp.ko", // TODO(b/176860479) once virt_wifi is deprecated fully, // these following modules can be loaded in second stage init "libarc4.ko", diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/doc/vm.dot b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/doc/vm.dot index 9b998fa9133..4212c5b055b 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/doc/vm.dot +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/doc/vm.dot @@ -16,6 +16,7 @@ graph { vm--config vm--memory_mb vm--custom_actions + vm--enable_pkvm vm--vm_manager crosvm_binary_dir [label = "binary_dir"] qemu_binary_dir [label = "binary_dir"] diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/doc/vm.png b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/doc/vm.png index 6fbb93215eb..d4f063f1850 100644 Binary files a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/doc/vm.png and b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/doc/vm.png differ diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/doc/vm.svg b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/doc/vm.svg index 7a54f7d5f58..0dcbd63a445 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/doc/vm.svg +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/doc/vm.svg @@ -1,474 +1,486 @@ - - - - - + + + +%3 + vm - -vm + +vm cpus - -cpus + +cpus vm--cpus - + vm_manager - -vm_manager + +vm_manager vm--vm_manager - + - + vm--vm_manager - + vsock_guest_cid - -vsock_guest_cid + +vsock_guest_cid vm--vsock_guest_cid - + enable_minimal_mode - -enable_minimal_mode + +enable_minimal_mode vm--enable_minimal_mode - + restart_subprocesses - -restart_subprocesses + +restart_subprocesses vm--restart_subprocesses - + setupwizard_mode - -setupwizard_mode + +setupwizard_mode vm--setupwizard_mode - + smt - -smt + +smt vm--smt - + use_allocd - -use_allocd + +use_allocd vm--use_allocd - + use_sdcard - -use_sdcard + +use_sdcard vm--use_sdcard - + uuid - -uuid + +uuid vm--uuid - + file_verbosity - -file_verbosity + +file_verbosity vm--file_verbosity - + verbosity - -verbosity + +verbosity vm--verbosity - + Run_file_discovery - -Run_file_discovery + +Run_file_discovery vm--Run_file_discovery - + config - -config + +config vm--config - + memory_mb - -memory_mb + +memory_mb vm--memory_mb - + custom_actions - -custom_actions + +custom_actions vm--custom_actions - + + + + +enable_pkvm + +enable_pkvm + + + +vm--enable_pkvm + - + security - -security + +security - + vm--security - + - + kernel - -kernel + +kernel - + vm--kernel - + - + crosvm - -crosvm + +crosvm - + vm_manager--crosvm - + - + qemu - -qemu + +qemu - + vm_manager--qemu - + - + gem5 - -gem5 + +gem5 - + vm_manager--gem5 - + - + crosvm_binary_dir - -binary_dir + +binary_dir - + qemu_binary_dir - -binary_dir + +binary_dir - + gem5_binary_dir - -binary_dir + +binary_dir - + crosvm--crosvm_binary_dir - + - + seccomp_policy_dir - -seccomp_policy_dir + +seccomp_policy_dir - + crosvm--seccomp_policy_dir - + - + enable_sandbox - -enable_sandbox + +enable_sandbox - + crosvm--enable_sandbox - + - + simple_media_device - -simple_media_device + +simple_media_device - + crosvm--simple_media_device - + - + v4l2_proxy - -v4l2_proxy + +v4l2_proxy - + crosvm--v4l2_proxy - + - + qemu--qemu_binary_dir - + - + gem5--gem5_binary_dir - + - + checkpoint_dir - -checkpoint_dir + +checkpoint_dir - + gem5--checkpoint_dir - + - + debug_file - -debug_file + +debug_file - + gem5--debug_file - + - + debug_flags - -debug_flags + +debug_flags - + gem5--debug_flags - + - + guest_enforce_security - -guest_enforce_security + +guest_enforce_security - + security--guest_enforce_security - + - + serial_number - -serial_number + +serial_number - + security--serial_number - + - + secure_hals - -secure_hals + +secure_hals - + security--secure_hals - + - + enable_kernel_log - -enable_kernel_log + +enable_kernel_log - + kernel--enable_kernel_log - + - + kgdb - -kgdb + +kgdb - + kernel--kgdb - + - + gdb_port - -gdb_port + +gdb_port - + kernel--gdb_port - + - + console - -console + +console - + kernel--console - + - + extra_kernel_cmdline - -extra_kernel_cmdline + +extra_kernel_cmdline - + kernel--extra_kernel_cmdline - + - + initramfs_path - -initramfs_path + +initramfs_path - + kernel--initramfs_path - + - + path - -path + +path - + kernel--path - + diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/golang/load_config.pb.go b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/golang/load_config.pb.go index 8006029776b..80041941764 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/golang/load_config.pb.go +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/golang/load_config.pb.go @@ -141,15 +141,16 @@ func (ModemSimulatorSimType) EnumDescriptor() ([]byte, []int) { } type EnvironmentSpecification struct { - state protoimpl.MessageState `protogen:"open.v1"` - Instances []*Instance `protobuf:"bytes,1,rep,name=instances,proto3" json:"instances,omitempty"` - Fetch *Fetch `protobuf:"bytes,2,opt,name=fetch,proto3,oneof" json:"fetch,omitempty"` - Metrics *Metrics `protobuf:"bytes,3,opt,name=metrics,proto3,oneof" json:"metrics,omitempty"` - Common *Common `protobuf:"bytes,4,opt,name=common,proto3,oneof" json:"common,omitempty"` - NetsimBt *bool `protobuf:"varint,5,opt,name=netsim_bt,json=netsimBt,proto3,oneof" json:"netsim_bt,omitempty"` - NetsimUwb *bool `protobuf:"varint,6,opt,name=netsim_uwb,json=netsimUwb,proto3,oneof" json:"netsim_uwb,omitempty"` - NetsimArgs []string `protobuf:"bytes,7,rep,name=netsim_args,json=netsimArgs,proto3" json:"netsim_args,omitempty"` - NetsimModem *bool `protobuf:"varint,8,opt,name=netsim_modem,json=netsimModem,proto3,oneof" json:"netsim_modem,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Instances []*Instance `protobuf:"bytes,1,rep,name=instances,proto3" json:"instances,omitempty"` + Fetch *Fetch `protobuf:"bytes,2,opt,name=fetch,proto3,oneof" json:"fetch,omitempty"` + // deprecated v1 metrics enabling field, is not read/used + Metrics *Metrics `protobuf:"bytes,3,opt,name=metrics,proto3,oneof" json:"metrics,omitempty"` + Common *Common `protobuf:"bytes,4,opt,name=common,proto3,oneof" json:"common,omitempty"` + NetsimBt *bool `protobuf:"varint,5,opt,name=netsim_bt,json=netsimBt,proto3,oneof" json:"netsim_bt,omitempty"` + NetsimUwb *bool `protobuf:"varint,6,opt,name=netsim_uwb,json=netsimUwb,proto3,oneof" json:"netsim_uwb,omitempty"` + NetsimArgs []string `protobuf:"bytes,7,rep,name=netsim_args,json=netsimArgs,proto3" json:"netsim_args,omitempty"` + NetsimModem *bool `protobuf:"varint,8,opt,name=netsim_modem,json=netsimModem,proto3,oneof" json:"netsim_modem,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -243,14 +244,15 @@ func (x *EnvironmentSpecification) GetNetsimModem() bool { // TODO: chadreynolds - when we can make breaking changes and update, use this // in EnvironmentSpecification instead of individual fields type EnvironmentOptions struct { - state protoimpl.MessageState `protogen:"open.v1"` - Fetch *Fetch `protobuf:"bytes,1,opt,name=fetch,proto3,oneof" json:"fetch,omitempty"` - Metrics *Metrics `protobuf:"bytes,2,opt,name=metrics,proto3,oneof" json:"metrics,omitempty"` - Common *Common `protobuf:"bytes,3,opt,name=common,proto3,oneof" json:"common,omitempty"` - NetsimBt *bool `protobuf:"varint,4,opt,name=netsim_bt,json=netsimBt,proto3,oneof" json:"netsim_bt,omitempty"` - NetsimUwb *bool `protobuf:"varint,5,opt,name=netsim_uwb,json=netsimUwb,proto3,oneof" json:"netsim_uwb,omitempty"` - NetsimArgs []string `protobuf:"bytes,6,rep,name=netsim_args,json=netsimArgs,proto3" json:"netsim_args,omitempty"` - NetsimModem *bool `protobuf:"varint,7,opt,name=netsim_modem,json=netsimModem,proto3,oneof" json:"netsim_modem,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Fetch *Fetch `protobuf:"bytes,1,opt,name=fetch,proto3,oneof" json:"fetch,omitempty"` + // deprecated v1 metrics enabling field, is not read/used + Metrics *Metrics `protobuf:"bytes,2,opt,name=metrics,proto3,oneof" json:"metrics,omitempty"` + Common *Common `protobuf:"bytes,3,opt,name=common,proto3,oneof" json:"common,omitempty"` + NetsimBt *bool `protobuf:"varint,4,opt,name=netsim_bt,json=netsimBt,proto3,oneof" json:"netsim_bt,omitempty"` + NetsimUwb *bool `protobuf:"varint,5,opt,name=netsim_uwb,json=netsimUwb,proto3,oneof" json:"netsim_uwb,omitempty"` + NetsimArgs []string `protobuf:"bytes,6,rep,name=netsim_args,json=netsimArgs,proto3" json:"netsim_args,omitempty"` + NetsimModem *bool `protobuf:"varint,7,opt,name=netsim_modem,json=netsimModem,proto3,oneof" json:"netsim_modem,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1462,6 +1464,7 @@ type Vm struct { CustomActions []*CustomAction `protobuf:"bytes,9,rep,name=custom_actions,json=customActions,proto3" json:"custom_actions,omitempty"` // Desired user space page size. PageSize *UserPageSize `protobuf:"varint,10,opt,name=page_size,json=pageSize,proto3,enum=cuttlefish.cvd.config.UserPageSize,oneof" json:"page_size,omitempty"` + EnablePkvm *bool `protobuf:"varint,11,opt,name=enable_pkvm,json=enablePkvm,proto3,oneof" json:"enable_pkvm,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1579,6 +1582,13 @@ func (x *Vm) GetPageSize() UserPageSize { return UserPageSize_USER_PAGE_SIZE_UNSPECIFIED } +func (x *Vm) GetEnablePkvm() bool { + if x != nil && x.EnablePkvm != nil { + return *x.EnablePkvm + } + return false +} + type isVm_Vmm interface { isVm_Vmm() } @@ -2198,7 +2208,7 @@ const file_cuttlefish_host_commands_cvd_cli_parser_load_config_proto_rawDesc = " "\tV4l2Proxy\x12$\n" + "\vdevice_path\x18\x01 \x01(\tH\x00R\n" + "devicePath\x88\x01\x01B\x0e\n" + - "\f_device_path\"\xb7\x04\n" + + "\f_device_path\"\xed\x04\n" + "\x02Vm\x12\x17\n" + "\x04cpus\x18\x01 \x01(\rH\x01R\x04cpus\x88\x01\x01\x12 \n" + "\tmemory_mb\x18\x02 \x01(\rH\x02R\bmemoryMb\x88\x01\x01\x12\"\n" + @@ -2211,7 +2221,9 @@ const file_cuttlefish_host_commands_cvd_cli_parser_load_config_proto_rawDesc = " "\x04qemu\x18\b \x01(\v2\x1b.cuttlefish.cvd.config.QemuH\x00R\x04qemu\x12J\n" + "\x0ecustom_actions\x18\t \x03(\v2#.cuttlefish.cvd.config.CustomActionR\rcustomActions\x12E\n" + "\tpage_size\x18\n" + - " \x01(\x0e2#.cuttlefish.cvd.config.UserPageSizeH\x06R\bpageSize\x88\x01\x01B\x05\n" + + " \x01(\x0e2#.cuttlefish.cvd.config.UserPageSizeH\x06R\bpageSize\x88\x01\x01\x12$\n" + + "\venable_pkvm\x18\v \x01(\bH\aR\n" + + "enablePkvm\x88\x01\x01B\x05\n" + "\x03vmmB\a\n" + "\x05_cpusB\f\n" + "\n" + @@ -2220,7 +2232,8 @@ const file_cuttlefish_host_commands_cvd_cli_parser_load_config_proto_rawDesc = " "\x11_setupwizard_modeB\a\n" + "\x05_uuidB\f\n" + "\n" + - "_page_size\"\x8b\x02\n" + + "_page_sizeB\x0e\n" + + "\f_enable_pkvm\"\x8b\x02\n" + "\x06Crosvm\x12*\n" + "\x0eenable_sandbox\x18\x01 \x01(\bH\x00R\renableSandbox\x88\x01\x01\x123\n" + "\x13simple_media_device\x18\x02 \x01(\bH\x01R\x11simpleMediaDevice\x88\x01\x01\x12\"\n" + diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/instance/cf_vm_configs.cpp b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/instance/cf_vm_configs.cpp index c30bb981dbf..181707a67ff 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/instance/cf_vm_configs.cpp +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/instance/cf_vm_configs.cpp @@ -49,6 +49,7 @@ inline constexpr char kFlagEnableSandbox[] = "enable_sandbox"; inline constexpr char kFlagCrosvmSimpleMediaDevice[] = "crosvm_simple_media_device"; inline constexpr char kFlagCrosvmV4l2Proxy[] = "crosvm_v4l2_proxy"; +inline constexpr char kFlagEnablePkvm[] = "enable_pkvm"; std::set GatherFlagNamesUsedInInstanceConfig(const Instance& ins) { std::set names; @@ -82,6 +83,9 @@ std::set GatherFlagNamesUsedInInstanceConfig(const Instance& ins) { ins.vm().crosvm().has_v4l2_proxy()) { names.insert(kFlagCrosvmV4l2Proxy); } + if (ins.vm().has_enable_pkvm()) { + names.insert(kFlagEnablePkvm); + } return names; } @@ -175,6 +179,11 @@ static std::string V4l2Proxy(const Instance& instance) { return crosvm.has_v4l2_proxy() ? crosvm.v4l2_proxy() : default_val; } +static bool EnablePkvm(const Instance& instance) { + const auto& vm = instance.vm(); + return vm.has_enable_pkvm() ? vm.enable_pkvm() : CF_DEFAULTS_ENABLE_PKVM; +} + static std::vector UserPageSize( const EnvironmentSpecification& cfg) { std::vector ret; @@ -260,6 +269,9 @@ Result> GenerateVmFlags( flags.emplace_back( GenerateInstanceFlag(kFlagCrosvmV4l2Proxy, cfg, V4l2Proxy)); } + if (used_names.contains(kFlagEnablePkvm)) { + flags.emplace_back(GenerateInstanceFlag(kFlagEnablePkvm, cfg, EnablePkvm)); + } flags = MergeResults(std::move(flags), CF_EXPECT(CustomConfigsFlags(cfg))); flags = MergeResults(std::move(flags), UserPageSize(cfg)); diff --git a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/load_config.proto b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/load_config.proto index 3fbf253bf73..a9177160a87 100644 --- a/base/cvd/cuttlefish/host/commands/cvd/cli/parser/load_config.proto +++ b/base/cvd/cuttlefish/host/commands/cvd/cli/parser/load_config.proto @@ -192,6 +192,7 @@ message Vm { repeated CustomAction custom_actions = 9; // Desired user space page size. optional UserPageSize page_size = 10; + optional bool enable_pkvm = 11; } message Crosvm { diff --git a/base/cvd/cuttlefish/host/commands/start/main.cc b/base/cvd/cuttlefish/host/commands/start/main.cc index 3c7b8e29a91..36dc6100f22 100644 --- a/base/cvd/cuttlefish/host/commands/start/main.cc +++ b/base/cvd/cuttlefish/host/commands/start/main.cc @@ -153,6 +153,7 @@ const std::unordered_set& BoolFlags() { "enable_kernel_log", "enable_minimal_mode", "enable_modem_simulator", + "enable_pkvm", "enable_sandbox", "enable_usb", "enable_virtiofs", diff --git a/base/cvd/cuttlefish/host/libs/config/cuttlefish_config.h b/base/cvd/cuttlefish/host/libs/config/cuttlefish_config.h index 562e8ec9087..218d1df0d44 100644 --- a/base/cvd/cuttlefish/host/libs/config/cuttlefish_config.h +++ b/base/cvd/cuttlefish/host/libs/config/cuttlefish_config.h @@ -404,6 +404,7 @@ class CuttlefishConfig { bool crosvm_simple_media_device() const; std::string crosvm_v4l2_proxy() const; bool use_pmem() const; + bool enable_pkvm() const; // Wifi MAC address inside the guest int wifi_mac_prefix() const; @@ -656,6 +657,7 @@ class CuttlefishConfig { void set_crosvm_simple_media_device(const bool simple_media_device); void set_crosvm_v4l2_proxy(const std::string v4l2_proxy); void set_use_pmem(const bool use_pmem); + void set_enable_pkvm(bool enable_pkvm); // Wifi MAC address inside the guest void set_wifi_mac_prefix(const int wifi_mac_prefix); // Gnss grpc proxy server port inside the host diff --git a/base/cvd/cuttlefish/host/libs/config/cuttlefish_config_instance.cpp b/base/cvd/cuttlefish/host/libs/config/cuttlefish_config_instance.cpp index 68afec0d250..bafd2577e9b 100644 --- a/base/cvd/cuttlefish/host/libs/config/cuttlefish_config_instance.cpp +++ b/base/cvd/cuttlefish/host/libs/config/cuttlefish_config_instance.cpp @@ -1871,6 +1871,15 @@ bool CuttlefishConfig::InstanceSpecific::use_pmem() const { return (*Dictionary())[kCrosvmUsePmem].asBool(); } +static constexpr char kEnablePkvm[] = "enable_pkvm"; +void CuttlefishConfig::MutableInstanceSpecific::set_enable_pkvm( + bool enable_pkvm) { + (*Dictionary())[kEnablePkvm] = enable_pkvm; +} +bool CuttlefishConfig::InstanceSpecific::enable_pkvm() const { + return (*Dictionary())[kEnablePkvm].asBool(); +} + static constexpr char kEnableTapDevices[] = "enable_tap_devices"; void CuttlefishConfig::MutableInstanceSpecific::set_enable_tap_devices( const bool enable_tap_devices) { diff --git a/base/cvd/cuttlefish/host/libs/config/kernel_args.cpp b/base/cvd/cuttlefish/host/libs/config/kernel_args.cpp index 26561270951..342a4e1437a 100644 --- a/base/cvd/cuttlefish/host/libs/config/kernel_args.cpp +++ b/base/cvd/cuttlefish/host/libs/config/kernel_args.cpp @@ -109,6 +109,11 @@ std::vector KernelCommandLineFromConfig( const CuttlefishConfig::InstanceSpecific& instance) { std::vector kernel_cmdline; AppendVector(&kernel_cmdline, VmManagerKernelCmdline(config, instance)); + if (instance.enable_pkvm() && instance.target_arch() == Arch::Arm64) { + kernel_cmdline.push_back("kvm-arm.mode=protected"); + // Makes the guest kernel's early loader modprobe the pKVM IOMMU driver. + kernel_cmdline.push_back("kvm-arm.protected_modules=pkvm_iommu_temp"); + } AppendVector(&kernel_cmdline, config.extra_kernel_cmdline()); return kernel_cmdline; } diff --git a/base/cvd/cuttlefish/host/libs/vm_manager/crosvm_manager.cpp b/base/cvd/cuttlefish/host/libs/vm_manager/crosvm_manager.cpp index 4b36c2597f2..122e400ad8e 100644 --- a/base/cvd/cuttlefish/host/libs/vm_manager/crosvm_manager.cpp +++ b/base/cvd/cuttlefish/host/libs/vm_manager/crosvm_manager.cpp @@ -572,6 +572,12 @@ Result> CrosvmManager::StartCommands( crosvm_cmd.AddKvmPath(config.kvm_path()); } + // A pkvm guest needs to boot at virtual EL2; crosvm verifies host support + // (KVM_CAP_ARM_EL2, GICv3 irqchip) and fails to start otherwise. + if (instance.enable_pkvm()) { + crosvm_cmd.Cmd().AddParameter("--nested=on"); + } + if (!instance.smt()) { crosvm_cmd.Cmd().AddParameter("--no-smt"); } @@ -627,7 +633,14 @@ Result> CrosvmManager::StartCommands( } if (instance.hwcomposer() != kHwComposerNone) { - const bool pmem_disabled = instance.mte() || !instance.use_pmem(); + // pmem is disabled for pkvm guests: the pmem backing files are mmap'd + // MAP_SHARED, and host writeback of storage-backed pages triggers + // mmu-notifiers that arm64 KVM currently handles by tearing down the + // guest's entire shadow stage-2 instead of a range-scoped invalidation + // (see the nested_mmu reverse-mapping TODO in arch/arm64/kvm/mmu.c), + // which makes the guest extremely slow. + const bool pmem_disabled = + instance.mte() || instance.enable_pkvm() || !instance.use_pmem(); const std::string pmem_path = HwcomposerPmemPath(instance); if (!pmem_disabled && FileExists(pmem_path)) { crosvm_cmd.Cmd().AddParameter("--pmem=path=", pmem_path); @@ -714,7 +727,8 @@ Result> CrosvmManager::StartCommands( } #endif - const bool pmem_disabled = instance.mte() || !instance.use_pmem(); + const bool pmem_disabled = + instance.mte() || instance.enable_pkvm() || !instance.use_pmem(); const std::string access_kregistry = AccessKregistryPath(instance); if (!pmem_disabled && FileExists(access_kregistry)) { crosvm_cmd.Cmd().AddParameter("--pmem=path=", access_kregistry);