diff --git a/CHANGELOG.md b/CHANGELOG.md index a7c7c6ae7..fd6e60137 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added ### Changed +* **Breaking:** Retained guest MSR state is restored on snapshot restore. KVM + denies guest MSR reads and writes by default. MSHV and WHP reset all exposed + retained MSR state. `SandboxConfiguration::allow_msrs` permits specific MSRs + by @ludfjig in https://github.com/hyperlight-dev/hyperlight/pull/991 ### Removed diff --git a/Justfile b/Justfile index 1eb4dd3a0..d12fc7793 100644 --- a/Justfile +++ b/Justfile @@ -243,6 +243,10 @@ test-isolated target=default-target features="" : {{ cargo-cmd }} test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --test integration_test -- log_message --exact --ignored @# CPU vendor check, gated to known CI runner hardware {{ cargo-cmd }} test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --lib -- sandbox::snapshot::file::config::tests::cpu_vendor_current_is_recognized --exact --ignored + @# Slow host-dependent MSR audit. Run once per x86_64 CI profile. + {{ if features == "" { if hyperlight-target-arch == "x86_64" { cargo-cmd + " test --profile=" + (if target == "debug" { "dev" } else { target }) + " " + target-triple-flag + " -p hyperlight-host --lib -- sandbox::initialized_multi_use::tests::msr_tests::test_no_msr_leaks_across_restore_full_window_sweep --exact --ignored --nocapture" } else { "" } } else { "" } }} + @# LAPIC-enabled MSHV can expose additional MSRs. Audit it once in debug. + {{ if features == "mshv3,hw-interrupts" { if target == "debug" { if hyperlight-target-arch == "x86_64" { cargo-cmd + " test --no-default-features -F mshv3,hw-interrupts --profile=dev " + target-triple-flag + " -p hyperlight-host --lib -- sandbox::initialized_multi_use::tests::msr_tests::test_no_msr_leaks_across_restore_full_window_sweep --exact --ignored --nocapture" } else { "" } } else { "" } } else { "" } }} @# metrics tests {{ cargo-cmd }} test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F function_call_metrics," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --lib -- metrics::tests::test_metrics_are_emitted --exact diff --git a/docs/msr.md b/docs/msr.md new file mode 100644 index 000000000..316f097c5 --- /dev/null +++ b/docs/msr.md @@ -0,0 +1,304 @@ +# MSR state across restore + +## Requirement + +A snapshot restore must remove all model-specific register (MSR) state written +after the snapshot. The guest must observe the MSR values saved with the +restored state. + +## Reset set + +The reset set contains every MSR whose guest-written value can persist. Each +running snapshot stores values for this set. A snapshot created from a guest +binary has no saved MSR values, so restore uses the baseline captured when the +VM was created. + +`MSR_TABLE` lists the MSRs that hold retained state restore must write. A +write-only command MSR holds no state, so it is absent from the table. + +The resolved reset set contains the backend core set, required MTRRs, and the +validated allow list. Hyperlight sorts and deduplicates the indices before +capturing the initialization baseline. + +The required invariant is: + +```text +guest-writable retained state => host-readable and host-writable state +``` + +Host-readable state need not be guest-writable. Extra reset entries are safe. +`EFER`, `APIC_BASE`, `FS_BASE`, and `GS_BASE` belong to the special-register +state. + +The two halves are established differently. Resolution checks the +host-readable half at run time: a candidate index enters the set only when the +host read succeeds, so an unreadable MSR is dropped. Nothing checks the +host-writable half at run time. It holds by construction. Every reset MSR is +stored in VP register state that the Hyper-V host interface both reads and +writes, except `KERNEL_GS_BASE`, a real register the host reads and writes +directly. Round-trip tests plant a guest value, restore, and assert that it does +not survive. Every future entry must remain host-readable and host-writable. + +## Reset set justification + +Each entry is grounded in how the Hyper-V hypervisor handles a guest access to +that register, confirmed against the Hyper-V source. + +A register is reset when the guest can write it and Hyper-V keeps the written +value. Hyper-V keeps it in one of two ways: it stores the value in the VP's +saved register state, or it lets the guest write the real register directly. +Only `FS_BASE`, `GS_BASE`, and `KERNEL_GS_BASE` are written directly. Every +other register below is intercepted and stored. Hyper-V stores the value on both +Intel and AMD hosts. + +Interception alone is not the test. Hyper-V also intercepts registers the guest +can only read, or that return a host-derived value. Those keep no +guest-controlled state and are not reset. Each row below names the guest state +that persists. + +| MSR (index) | Retained guest state | +| --- | --- | +| SYSENTER CS, ESP, EIP (`0x174`-`0x176`) | Guest write retained. | +| STAR, LSTAR, CSTAR, SFMASK (`0xC000_0081`-`0xC000_0084`) | Guest write retained (syscall targets). | +| KERNEL_GS_BASE (`0xC000_0102`) | Guest write retained. Written to the real register, and reachable through `SWAPGS` without a `WRMSR`. | +| PAT (`0x277`) | Guest write retained. | +| DEBUGCTL (`0x1D9`) | Guest write retained. | +| SPEC_CTRL (`0x48`) | Guest write retained. | +| CET U_CET, S_CET, PL0-3_SSP, INTERRUPT_SSP_TABLE_ADDR (`0x6A0`, `0x6A2`, `0x6A4`-`0x6A8`) | Guest write retained. | +| XSS (`0xDA0`) | Guest write retained. | +| TSC (`0x10`) | Guest write retained. Hyper-V forbids intercepting its implemented TSC, so restore rewrites the captured value. | +| TSC_ADJUST (`0x3B`) | Guest write retained, independent of TSC. | +| TSC_AUX (`0xC000_0103`) | Guest write retained. | +| MTRRs (`0x2FF`, `0x200`-`0x21F`, `0x250`, `0x258`-`0x259`, `0x268`-`0x26F`) | Guest write retained (memory-type state). | +| TSX_CTRL (`0x122`) | Guest write retained. | +| XFD, XFD_ERR (`0x1C4`, `0x1C5`) | Guest write retained. | +| UMWAIT_CONTROL (`0xE1`) | Guest write retained. Intel only. | +| TSC_DEADLINE (`0x6E0`) | Guest write retained. | +| BNDCFGS (`0xD90`) | Guest write retained when the host supports MPX. A guest access faults otherwise. | +| MPERF, APERF (`0xE7`, `0xE8`) | Guest write retained in per-VP counters. | + +Write-only command MSRs hold no state. A guest write performs an action and +leaves nothing to restore, so they are absent from the reset table and cannot +be allowed. + +| MSR (index) | Behavior | +| --- | --- | +| PRED_CMD (`0x49`) | Guest write issues a prediction barrier. | +| FLUSH_CMD (`0x10B`) | Guest write flushes caches. | + +Some registers are deliberately absent because guest access cannot leave state +outside another reset mechanism. + +| MSR (index) | Why excluded | +| --- | --- | +| MISC_ENABLE (`0x1A0`) | Intercepted, but Hyper-V discards a guest write and returns a fixed value. No retained state. On AMD the access faults. See below. | +| FRED (`0x1CC`-`0x1D4`) | Retained only when the host exposes FRED, which Hyperlight does not. A guest access faults otherwise. | +| PASID (`0xD93`) | MSHV exposes ENQCMD on capable Intel hosts. PASID is a supervisor XSAVE component, so XSAVE reset clears it. WHP does not expose ENQCMD. | +| PMU: PMC0, PERFEVTSEL0, FIXED_CTR_CTRL, PERF_GLOBAL_CTRL (`0xC1`, `0x186`, `0x38D`, `0x38F`) | Heads of the performance-monitoring class. Hyper-V leaves these unimplemented for the guest and installs guest-accessible descriptors, sized to the CPU counter count, only when perfmon is enabled. Hyperlight enables no perfmon, so a guest access faults and retains nothing. | +| LBR: LBR_SELECT, LASTBRANCH_TOS, LBR_CTL, LBR_DEPTH (`0x1C8`, `0x1C9`, `0x14CE`, `0x14CF`) | Last-branch registers, gated with perfmon. A guest access faults and retains nothing while perfmon stays off. | + +Hyper-V virtualizes `BNDCFGS`, `FRED`, and `PASID` only when the matching CPU +feature is exposed. `BNDCFGS` is reset because MPX is exposed by default on +capable hosts. `FRED` stays excluded because Hyperlight does not expose it. +MSHV can expose ENQCMD, but its XSAVE state mask then includes PASID. Hyperlight +clears PASID during XSAVE reset before restoring MSRs. The performance-monitoring +and last-branch registers remain inaccessible while perfmon is off. + +## Snapshot validation + +Snapshot MSR entries are untrusted. A snapshot records the reset values and the +capturing sandbox's allow list. `validate_snapshot` enforces two rules against +the destination VM's reset set: + +* The snapshot's allow list must be a subset of the destination's. A + destination that allows at least as much accepts the snapshot. +* Every supplied index must belong to the destination reset set. + +Indices the destination resets but the snapshot omits take the destination's +creation-time baseline. A rejected restore poisons the sandbox before the guest +can run. Equivalent allow lists produce the same sorted reset set, regardless of +insertion order. + +## Restore across allow lists + +A restore or `from_snapshot` succeeds when the destination allow list is a +superset of the snapshot's, on every backend. The snapshot's allowed MSRs keep +their captured values. An MSR the destination allows but the snapshot did not +resets to the destination baseline. A non-superset allow list is rejected +uniformly. + +The rule is backend independent even though each backend sizes its reset set +differently. KVM derives its reset set from the allow list. MSHV and WHP reset +the full host table. The allow-list subset check gates the restore before either +reset set is applied, so a flow that succeeds on one backend succeeds on all. + +The superset check is the common rule across backends. MSHV and WHP accept any +allow list on their own. The shared check gives every backend KVM's constraint. + +## Allow list + +`SandboxConfiguration::allow_msrs` adds indices to the requested allow list. It +enforces capacity only. VM creation verifies that each index is resettable and +supported by the selected backend. + +KVM requires the index in `KVM_GET_MSR_INDEX_LIST` and a successful host read +and write. MSHV and WHP require a named-register mapping and a successful host +read. + +At most 16 distinct MSRs may be requested. KVM also limits the resulting +contiguous filter groups to 16. + +## KVM + +KVM installs a deny filter over the full MSR space. Allowed indices form the +only guest `RDMSR` and `WRMSR` paths through that filter. A denied access exits +to Hyperlight, injects `#GP`, and poisons the sandbox. The denied write stores +no state. + +The KVM reset set contains the allow list plus `KERNEL_GS_BASE` and `TSC`. +`KERNEL_GS_BASE` is required because `WRGSBASE` followed by `SWAPGS` changes it +without `WRMSR`. `TSC` gives restore the same clock semantics on every backend. + +KVM does not filter x2APIC indices `0x800..=0x8FF`. Hyperlight keeps the APIC in +xAPIC mode, where MSR access to that range raises `#GP`. `APIC_BASE` is not an +allowable MSR, so a guest cannot enable x2APIC. Snapshots created by Hyperlight +therefore retain `APIC_BASE.EXTD = 0`. File snapshots serialize `APIC_BASE` +without semantic validation, so the caller must trust the snapshot source as +required by the snapshot format. + +## MTRRs + +MSHV and WHP read `IA32_MTRRCAP` when the VM is created. The required set +contains `MTRR_DEF_TYPE`, each variable pair reported by `VCNT`, and all fixed +MTRRs. + +Hyper-V accepts fixed-MTRR writes even when `MTRRCAP.FIX` is clear. All fixed +MTRRs are therefore required. Hyper-V supports at most 16 variable pairs. VM +creation fails when the count is larger or a required MTRR cannot be read. + +## MSHV + +MSHV has no per-MSR filter. Hyper-V permits an MSR intercept only for an +unimplemented index, which already faults for the guest, and cannot intercept +the implemented MSRs that hold retained state. Isolation therefore comes from +reset, not a deny filter. + +The MSHV reset set contains every table entry that has a Hyper-V +register mapping and can be read, plus the allow list. + +`msr_to_hv_reg_name` determines which indices the get and set path can reach. +The enumerated host index list does not identify retained state, so it does not +define the reset set. + +MSHV maps `IA32_XSS` through `MSR_IA32_REGISTER_U_XSS`. It maps `IA32_MPERF` +and `IA32_APERF` to the per-VP `MCount` and `ACount` registers. TSX control, +XFD, MPX (`BNDCFGS`), WAITPKG (`UMWAIT_CONTROL`), and the TSC deadline timer +enter the reset set when their host-register probes succeed. + +MSHV enables every host-supported processor feature unless the caller supplies +an explicit disabled-feature mask. Hyperlight supplies no mask. On capable +Intel hosts this can expose ENQCMD and its PASID MSR. MSHV reports PASID in the +partition XSAVE state mask, and Hyperlight's XSAVE reset clears it. + +## WHP + +WHP has no per-MSR filter. Its reset set contains every table entry +that has a WHP register name and can be read, plus the allow list. + +WHP uses Germanium compatibility. Speculation control is off in its default +feature banks, and perfmon (the PMU and architectural LBR) is a separate +property WHP leaves off. Experimental `DEBUGCTL` bits stay disabled. The WHP +API defines no FRED feature and its supported feature mask omits ENQCMD, so WHP +cannot expose FRED or PASID. + +Each guest MSR write is either captured for restore or unsupported by the +partition. Unsupported writes store no state. + +## TSC + +MSHV and WHP expose `TSC` as a host-writable register. Hyper-V stores `TSC` and +`TSC_ADJUST` independently, so restoring `TSC_ADJUST` cannot undo a guest +`WRMSR(TSC)`. + +While time is running, Hyper-V preserves `TSC - TSC_ADJUST`: writing `TSC` +adds the same delta to `TSC_ADJUST`, and writing `TSC_ADJUST` adds its delta to +the internal TSC offset. Restoring `TSC` followed by `TSC_ADJUST` therefore +cancels any guest-controlled delta. Freezing partition time is not required for +isolation. + +Hyper-V does not permit an intercept for its implemented `TSC` MSR. Restore +must therefore write the captured `TSC` value. KVM also restores `TSC` so all +backends rewind guest time with the rest of the snapshot state. + +## Feature exposure + +On MSHV and WHP a guest reaches an MSR only when the hypervisor exposes that +CPU feature to the partition. This gives three cases: + +* Not exposed. Features the partition does not enable, such as the + performance-monitoring unit, last-branch records, and FRED. Hyper-V may still + model the register, but a guest access faults and stores no state until the + feature is exposed. +* Exposed by default. Features the host CPU supports, such as TSC deadline, + UMWAIT, TSX control, CET, `MPERF`/`APERF`, XFD, AMX, and MPX. Their MSRs + must be in the reset set. +* Reset through another state class. MSHV can expose ENQCMD and PASID on + capable Intel hosts. PASID is cleared by XSAVE reset, so it is not duplicated + in the MSR reset set. + +MSHV and WHP enable partition features differently. MSHV creates the partition +without an explicit feature mask, so it enables every processor feature the host +supports. WHP starts from the host-supported set with speculation control off. +MSHV exposes the broader surface and determines which registers the reset set +must cover. + +Perfmon is not part of either default. The performance-monitoring unit and the +last-branch registers are a separate opt-in partition property, off by default +on both backends. Hyperlight never enables it, so those registers stay +unreachable regardless of the enable-everything processor-feature default. + +Only reachable, retained MSRs need coverage, and retained state is always held +in a host-readable and writable register. The mapped registers therefore bound +the reset set: coverage is complete when every mapped register is in the reset +table and reset. + +## Host-addressable but not guest-writable + +A host register mapping does not imply the guest can write the MSR. +`IA32_MISC_ENABLE` (`0x1A0`) is the notable case. Hyper-V emulates it, discards +a guest write, and returns a fixed value to the guest regardless of what was +written. A guest cannot change it to any value, so it retains no guest state +and needs no reset. On AMD the guest access faults. + +## Failed access reporting + +A KVM-denied or Hyper-V-unsupported MSR access does not persist and poisons the +sandbox. The error type and its detail differ by backend. + +* KVM traps the access at the deny filter. Hyperlight reports + `MsrReadViolation` or `MsrWriteViolation`, naming the MSR index and, for a + write, the value. The report is host-verified. +* MSHV and WHP have no host MSR trap. An unsupported access faults inside the + guest as a general protection fault from Hyper-V, so Hyperlight reports + `GuestAborted`. The message records the fault and the faulting instruction but + does not identify the MSR. An exposed MSR can succeed even when absent from + the allow list. Its retained state must be in the reset table. + +Future work: the guest exception handler could decode a faulting `RDMSR` or +`WRMSR` and report the index, promoting the abort to a typed MSR violation on +MSHV and WHP. That index would be guest-reported and therefore advisory. It is +not implemented. + +## Limitations + +KVM's security boundary is structural because its deny filter bounds guest +writes. MSHV and WHP depend on the reset table and exposed processor +features. + +The filterless backend tests run on one CPU model per runner. Model-specific +state absent on that CPU is not exercised. A backend that exposes a new +retained MSR feature needs a matching table entry before Hyperlight can use it +safely. + +The ignored full-window audit probes fixed index ranges with a small set of +values. It cannot prove that every vendor MSR or accepted value is covered. diff --git a/docs/snapshot-versioning.md b/docs/snapshot-versioning.md index ec3dca233..0d465f651 100644 --- a/docs/snapshot-versioning.md +++ b/docs/snapshot-versioning.md @@ -44,6 +44,18 @@ The config blob also records `hyperlight_version`, the `CARGO_PKG_VERSION` of the host crate at write time. This is informational only. The loader records it for diagnostics and does not gate loading on it. +## Compatibility cleanup + +Record compatibility paths here when a future hard snapshot break can remove +them. + +### Optional MSR fields + +The persisted `msrs` and `allowed_msrs` fields are optional so snapshots made +before MSR capture remain loadable. At the next hard break, make them required +vectors and remove `serde(default)` and the missing-field fallback. Keep the +in-memory fields optional while `Snapshot` represents pre-init state. + ## Enforcement The format is large and easy to change by accident. Two mechanisms diff --git a/src/hyperlight_host/src/error.rs b/src/hyperlight_host/src/error.rs index c6738374d..0b982eed6 100644 --- a/src/hyperlight_host/src/error.rs +++ b/src/hyperlight_host/src/error.rs @@ -154,6 +154,16 @@ pub enum HyperlightError { #[error("Memory Access Violation at address {0:#x} of type {1}, but memory is marked as {2}")] MemoryAccessViolation(u64, MemoryRegionFlags, MemoryRegionFlags), + /// A denied guest MSR read. + #[cfg(all(target_arch = "x86_64", kvm))] + #[error("Guest read from denied MSR {0:#x}")] + MsrReadViolation(u32), + + /// A denied guest MSR write. + #[cfg(all(target_arch = "x86_64", kvm))] + #[error("Guest write of {1:#x} to denied MSR {0:#x}")] + MsrWriteViolation(u32, u64), + /// Memory Allocation Failed. #[error("Memory Allocation Failed with OS Error {0:?}.")] MemoryAllocationFailed(Option), @@ -352,6 +362,10 @@ impl HyperlightError { // as poisoning here too for defense in depth. | HyperlightError::HyperlightVmError(HyperlightVmError::Restore(_)) => true, + #[cfg(all(target_arch = "x86_64", kvm))] + HyperlightError::MsrReadViolation(_) + | HyperlightError::MsrWriteViolation(_, _) => true, + // These errors poison the sandbox because they can leave // it in an inconsistent state due to snapshot restore // failing partway through diff --git a/src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs b/src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs index 56f7d635d..2d9ad6c3e 100644 --- a/src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs +++ b/src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs @@ -163,6 +163,16 @@ impl DispatchGuestCallError { region_flags, }) => HyperlightError::MemoryAccessViolation(addr, access_type, region_flags), + #[cfg(all(target_arch = "x86_64", kvm))] + DispatchGuestCallError::Run(RunVmError::MsrReadViolation(msr_index)) => { + HyperlightError::MsrReadViolation(msr_index) + } + + #[cfg(all(target_arch = "x86_64", kvm))] + DispatchGuestCallError::Run(RunVmError::MsrWriteViolation { msr_index, value }) => { + HyperlightError::MsrWriteViolation(msr_index, value) + } + // Leave others as is other => HyperlightVmError::DispatchGuestCall(other).into(), }; @@ -213,6 +223,12 @@ pub enum RunVmError { MmioReadUnmapped(u64), #[error("MMIO WRITE access to unmapped address {0:#x}")] MmioWriteUnmapped(u64), + #[cfg(all(target_arch = "x86_64", kvm))] + #[error("Guest read from denied MSR {0:#x}")] + MsrReadViolation(u32), + #[cfg(all(target_arch = "x86_64", kvm))] + #[error("Guest write of {value:#x} to denied MSR {msr_index:#x}")] + MsrWriteViolation { msr_index: u32, value: u64 }, #[error("vCPU run failed: {0}")] RunVcpu(#[from] RunVcpuError), #[error("Unexpected VM exit: {0}")] @@ -409,6 +425,9 @@ pub(crate) struct HyperlightVm { pub(super) trace_info: MemTraceInfo, #[cfg(crashdump)] pub(super) rt_cfg: SandboxRuntimeConfig, + /// MSRs restored on snapshot restore. + #[cfg(target_arch = "x86_64")] + pub(super) msr_reset: crate::hypervisor::regs::MsrResetState, } impl HyperlightVm { @@ -732,6 +751,14 @@ impl HyperlightVm { } } } + #[cfg(all(target_arch = "x86_64", kvm))] + Ok(VmExit::MsrRead(msr_index)) => { + break Err(RunVmError::MsrReadViolation(msr_index)); + } + #[cfg(all(target_arch = "x86_64", kvm))] + Ok(VmExit::MsrWrite { msr_index, value }) => { + break Err(RunVmError::MsrWriteViolation { msr_index, value }); + } Ok(VmExit::Cancelled()) => { // If cancellation was not requested for this specific guest function call, // the vcpu was interrupted by a stale cancellation. This can occur when: diff --git a/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs b/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs index 82206f787..408c7812c 100644 --- a/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs @@ -41,9 +41,12 @@ use crate::hypervisor::gdb::{ #[cfg(gdb)] use crate::hypervisor::gdb::{DebugError, DebugMemoryAccessError}; use crate::hypervisor::regs::{ - CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters, + CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters, MsrEntry, MsrResetState, + core_reset_indices, is_mtrr_reset_index, }; -#[cfg(not(gdb))] +#[cfg(kvm)] +use crate::hypervisor::regs::{MSR_KERNEL_GS_BASE, MSR_TSC}; +#[cfg(any(not(gdb), mshv3, target_os = "windows"))] use crate::hypervisor::virtual_machine::VirtualMachine; #[cfg(kvm)] use crate::hypervisor::virtual_machine::kvm::KvmVm; @@ -69,6 +72,26 @@ use crate::sandbox::trace::MemTraceInfo; #[cfg(crashdump)] use crate::sandbox::uninitialized::SandboxRuntimeConfig; +#[cfg(gdb)] +type BoxedVm = Box; +#[cfg(not(gdb))] +type BoxedVm = Box; + +/// Determines the MTRR reset indices for an MSHV or WHP backend and validates +/// its allow list. Both hosts lack an MSR filter, so the allow list adds reset +/// state. +#[cfg(any(mshv3, target_os = "windows"))] +fn determine_reset_msrs( + vm: &dyn VirtualMachine, + allowed: &[u32], +) -> std::result::Result, VmError> { + use crate::hypervisor::virtual_machine::{mtrr_reset_indices, validate_allowed_msrs}; + + let required_mtrrs = mtrr_reset_indices(vm).map_err(VmError::CreateVm)?; + validate_allowed_msrs(vm, allowed).map_err(VmError::CreateVm)?; + Ok(required_mtrrs) +} + impl HyperlightVm { /// Create a new HyperlightVm instance (will not run vm until calling `initialise`) #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")] @@ -85,18 +108,27 @@ impl HyperlightVm { #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig, #[cfg(feature = "mem_profile")] trace_info: MemTraceInfo, ) -> std::result::Result { - #[cfg(gdb)] - type VmType = Box; - #[cfg(not(gdb))] - type VmType = Box; - - let vm: VmType = match get_available_hypervisor() { + let (vm, required_mtrrs): (BoxedVm, Vec) = match get_available_hypervisor() { #[cfg(kvm)] - Some(HypervisorType::Kvm) => Box::new(KvmVm::new().map_err(VmError::CreateVm)?), + Some(HypervisorType::Kvm) => { + let kvm_vm = KvmVm::new().map_err(VmError::CreateVm)?; + kvm_vm + .configure_msr_access(config.get_allowed_msrs()) + .map_err(VmError::CreateVm)?; + (Box::new(kvm_vm), Vec::new()) + } #[cfg(mshv3)] - Some(HypervisorType::Mshv) => Box::new(MshvVm::new().map_err(VmError::CreateVm)?), + Some(HypervisorType::Mshv) => { + let vm: BoxedVm = Box::new(MshvVm::new().map_err(VmError::CreateVm)?); + let required_mtrrs = determine_reset_msrs(vm.as_ref(), config.get_allowed_msrs())?; + (vm, required_mtrrs) + } #[cfg(target_os = "windows")] - Some(HypervisorType::Whp) => Box::new(WhpVm::new().map_err(VmError::CreateVm)?), + Some(HypervisorType::Whp) => { + let vm: BoxedVm = Box::new(WhpVm::new().map_err(VmError::CreateVm)?); + let required_mtrrs = determine_reset_msrs(vm.as_ref(), config.get_allowed_msrs())?; + (vm, required_mtrrs) + } None => return Err(CreateHyperlightVmError::NoHypervisorFound), }; @@ -136,6 +168,10 @@ impl HyperlightVm { }), }); + // The MSRs reset on restore and captured in snapshots. + let msr_reset = + Self::capture_msr_reset_state(&vm, required_mtrrs, config.get_allowed_msrs())?; + let snapshot_slot = 0u32; let scratch_slot = 1u32; #[cfg_attr(not(gdb), allow(unused_mut))] @@ -168,6 +204,7 @@ impl HyperlightVm { trace_info, #[cfg(crashdump)] rt_cfg, + msr_reset, }; ret.update_snapshot_mapping(snapshot_mem)?; @@ -199,6 +236,50 @@ impl HyperlightVm { Ok(ret) } + /// Determines this VM's MSR reset set: the MSRs reset on restore and + /// captured in snapshots. `required_mtrrs` is the VCNT-driven MTRR set + /// gathered at creation. `allowed` is the validated allow list. + fn capture_msr_reset_state( + vm: &BoxedVm, + required_mtrrs: Vec, + allowed: &[u32], + ) -> std::result::Result { + let core: Vec = match get_available_hypervisor() { + #[cfg(kvm)] + Some(HypervisorType::Kvm) => vec![MSR_KERNEL_GS_BASE, MSR_TSC], // GS_BASE is needed for correctness. TSC is to match mshv/whp behavior + #[cfg(mshv3)] + Some(HypervisorType::Mshv) => Self::probe_core_reset_indices(vm), + #[cfg(target_os = "windows")] + Some(HypervisorType::Whp) => Self::probe_core_reset_indices(vm), + // The VM already exists, so a hypervisor was found. The `None` + // arm in `new` returned before this point. + None => unreachable!(), + }; + let mut indices: Vec = core + .into_iter() + .chain(required_mtrrs) + .chain(allowed.iter().copied()) + .collect(); + indices.sort_unstable(); + indices.dedup(); + let baseline = vm.msrs(&indices).map_err(VmError::Register)?; + let mut allowed = allowed.to_vec(); + allowed.sort_unstable(); + allowed.dedup(); + Ok(MsrResetState::new(baseline, allowed)) + } + + /// Core stateful MSRs a filterless host can read. + /// + /// MTRRs come from the VCNT-driven required set, so they are excluded. + #[cfg(any(mshv3, target_os = "windows"))] + fn probe_core_reset_indices(vm: &BoxedVm) -> Vec { + core_reset_indices() + .filter(|i| !is_mtrr_reset_index(*i)) + .filter(|i| vm.msrs(&[*i]).is_ok()) + .collect() + } + /// Initialise the internally stored vCPU with the given PEB address and /// random number seed, then run it until a HLT instruction. #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")] @@ -270,6 +351,54 @@ impl HyperlightVm { Ok(self.vm.sregs()?) } + /// Returns the current values of the MSR reset set. + pub(crate) fn get_msr_reset_state(&self) -> Result, AccessPageTableError> { + Ok(self.vm.msrs(&self.msr_reset.indices())?) + } + + /// Returns this VM's requested allow list, sorted and deduplicated. + pub(crate) fn get_msr_allow_list(&self) -> Vec { + self.msr_reset.allowed().to_vec() + } + + /// Restores snapshot MSRs or the initialization baseline. + pub(crate) fn restore_msrs( + &mut self, + snap_msrs: Option<&Vec>, + snap_allowed: Option<&[u32]>, + ) -> std::result::Result<(), ResetVcpuError> { + match snap_msrs { + // No captured MSRs. Use this VM's baseline. + None => self.vm.set_msrs(self.msr_reset.baseline())?, + // Reset the MSRs to the snapshot's captured values. The snapshot's + // allow list must be a subset of this VM's, and every captured + // index must be in this reset set, so validation rejects a + // mismatch first. + Some(msrs) => { + let entries = self + .msr_reset + .validate_snapshot(msrs, snap_allowed.unwrap_or(&[]))?; + self.vm.set_msrs(&entries)?; + } + } + Ok(()) + } + + /// Reads arbitrary backend MSRs for tests. + #[cfg(all(test, mshv3, target_arch = "x86_64"))] + pub(crate) fn capture_msrs_for_test( + &self, + indices: &[u32], + ) -> std::result::Result, RegisterError> { + self.vm.msrs(indices) + } + + /// Attempts one backend MSR write for tests. + #[cfg(all(test, mshv3, target_arch = "x86_64"))] + pub(crate) fn try_set_msr_for_test(&self, index: u32, value: u64) -> bool { + self.vm.set_msrs(&[MsrEntry { index, value }]).is_ok() + } + /// Dispatch a call from the host to the guest using the given pointer /// to the dispatch function _in the guest's address space_. /// diff --git a/src/hyperlight_host/src/hypervisor/regs/x86_64/mod.rs b/src/hyperlight_host/src/hypervisor/regs/x86_64/mod.rs index 88724d95a..12d65f056 100644 --- a/src/hyperlight_host/src/hypervisor/regs/x86_64/mod.rs +++ b/src/hyperlight_host/src/hypervisor/regs/x86_64/mod.rs @@ -16,11 +16,13 @@ limitations under the License. mod debug_regs; mod fpu; +mod msrs; mod special_regs; mod standard_regs; pub(crate) use debug_regs::*; pub(crate) use fpu::*; +pub(crate) use msrs::*; pub(crate) use special_regs::*; pub(crate) use standard_regs::*; diff --git a/src/hyperlight_host/src/hypervisor/regs/x86_64/msrs.rs b/src/hyperlight_host/src/hypervisor/regs/x86_64/msrs.rs new file mode 100644 index 000000000..86a688182 --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/regs/x86_64/msrs.rs @@ -0,0 +1,483 @@ +/* +Copyright 2025 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//! Model-specific register (MSR) state restored with a snapshot. +//! +//! The reset set contains every MSR whose guest-written state can persist. +//! Allowing guest access to an MSR is a separate concern. Allowed MSRs still +//! reset. + +use serde::{Deserialize, Serialize}; + +/// A single MSR captured for reset. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct MsrEntry { + /// The index passed to `RDMSR` and `WRMSR`. + pub index: u32, + /// The captured value. + pub value: u64, +} + +/// The MSR reset set and initialization baseline for one VM. +#[derive(Debug, Clone)] +pub(crate) struct MsrResetState { + /// Creation-time value for every MSR this VM resets on restore and + /// captures in a snapshot: core indices, required MTRRs, and the allow + /// list. Sorted by index and deduplicated. + baseline: Vec, + /// The user-supplied allow list (`SandboxConfiguration::allow_msrs`), + /// sorted and deduplicated. A subset of the `baseline` indices, kept as a + /// separate copy so restore can compare it, as a superset, against a + /// snapshot's allow list. + allowed: Vec, +} + +impl MsrResetState { + /// Builds the reset state from the creation-time baseline entries and the + /// requested allow list. + pub fn new(baseline: Vec, allowed: Vec) -> Self { + Self { baseline, allowed } + } + + /// The creation-time baseline entries. + pub fn baseline(&self) -> &[MsrEntry] { + &self.baseline + } + + /// This VM's requested allow list. + pub fn allowed(&self) -> &[u32] { + &self.allowed + } + + /// The MSR indices this VM resets, for host reads. + pub fn indices(&self) -> Vec { + self.baseline.iter().map(|entry| entry.index).collect() + } + + /// Resolves an untrusted snapshot's MSRs against this VM's reset set. + /// + /// The snapshot's allow list must be a subset of this VM's, so a + /// destination that allows at least as much accepts the snapshot on every + /// backend. Each supplied index must belong to this reset set. Returns the + /// entries to write: the snapshot's value for each reset index, or the + /// baseline where the snapshot omits it. + pub fn validate_snapshot( + &self, + snapshot: &[MsrEntry], + snap_allowed: &[u32], + ) -> Result, crate::hypervisor::virtual_machine::RegisterError> { + use crate::hypervisor::virtual_machine::RegisterError; + + let missing: Vec = snap_allowed + .iter() + .copied() + .filter(|index| !self.allowed.contains(index)) + .collect(); + if !missing.is_empty() { + return Err(RegisterError::SnapshotMsrNotAllowed { missing }); + } + + for entry in snapshot { + if !self.baseline.iter().any(|base| base.index == entry.index) { + return Err(RegisterError::InvalidSnapshotMsrIndex { index: entry.index }); + } + } + + Ok(self + .baseline + .iter() + .map(|base| MsrEntry { + index: base.index, + value: snapshot + .iter() + .find(|entry| entry.index == base.index) + .map_or(base.value, |entry| entry.value), + }) + .collect()) + } +} + +pub(crate) const MSR_TSC: u32 = 0x10; +pub(crate) const MSR_TSC_ADJUST: u32 = 0x3B; +pub(crate) const MSR_SPEC_CTRL: u32 = 0x48; +pub(crate) const MSR_UMWAIT_CONTROL: u32 = 0xE1; +pub(crate) const MSR_MPERF: u32 = 0xE7; +pub(crate) const MSR_APERF: u32 = 0xE8; +pub(crate) const MSR_MTRR_CAP: u32 = 0xFE; +pub(crate) const MSR_TSX_CTRL: u32 = 0x122; +pub(crate) const MSR_SYSENTER_CS: u32 = 0x174; +pub(crate) const MSR_SYSENTER_ESP: u32 = 0x175; +pub(crate) const MSR_SYSENTER_EIP: u32 = 0x176; +pub(crate) const MSR_XFD: u32 = 0x1C4; +pub(crate) const MSR_XFD_ERR: u32 = 0x1C5; +pub(crate) const MSR_DEBUGCTL: u32 = 0x1D9; +pub(crate) const MSR_MTRR_FIX64K_00000: u32 = 0x250; +pub(crate) const MSR_PAT: u32 = 0x277; +pub(crate) const MSR_MTRR_DEF_TYPE: u32 = 0x2FF; +pub(crate) const MSR_U_CET: u32 = 0x6A0; +pub(crate) const MSR_S_CET: u32 = 0x6A2; +pub(crate) const MSR_PL0_SSP: u32 = 0x6A4; +pub(crate) const MSR_PL1_SSP: u32 = 0x6A5; +pub(crate) const MSR_PL2_SSP: u32 = 0x6A6; +pub(crate) const MSR_PL3_SSP: u32 = 0x6A7; +pub(crate) const MSR_INTERRUPT_SSP_TABLE_ADDR: u32 = 0x6A8; +pub(crate) const MSR_TSC_DEADLINE: u32 = 0x6E0; +pub(crate) const MSR_BNDCFGS: u32 = 0xD90; +pub(crate) const MSR_XSS: u32 = 0xDA0; +pub(crate) const MSR_STAR: u32 = 0xC000_0081; +pub(crate) const MSR_LSTAR: u32 = 0xC000_0082; +pub(crate) const MSR_CSTAR: u32 = 0xC000_0083; +pub(crate) const MSR_SFMASK: u32 = 0xC000_0084; +pub(crate) const MSR_KERNEL_GS_BASE: u32 = 0xC000_0102; +pub(crate) const MSR_TSC_AUX: u32 = 0xC000_0103; +pub(crate) const MSR_VIRT_SPEC_CTRL: u32 = 0xC001_011F; + +const HYPERV_VARIABLE_MTRR_COUNT: u8 = 16; + +// Every guest-writable retained value must be host-readable and host-writable. +// EFER, APIC_BASE, FS_BASE, and GS_BASE are part of the special-register state. +// PRED_CMD (0x49) and FLUSH_CMD (0x10B) are intentionally absent: they are +// write-only commands with no retained state, so they cannot be reset and +// therefore cannot be allowed. +const MSR_TABLE: &[u32] = &[ + // Guest and host access use the matching SYSENTER state. + MSR_SYSENTER_CS, + MSR_SYSENTER_ESP, + MSR_SYSENTER_EIP, + // WHP exposes no writable DEBUGCTL bits under Germanium compatibility. + MSR_DEBUGCTL, + // Guest and host access use the same PAT state. + MSR_PAT, + // Guest and host access use the matching syscall state. + MSR_STAR, + MSR_LSTAR, + MSR_CSTAR, + MSR_SFMASK, + // SWAPGS and host access use the same KERNEL_GS_BASE state. + MSR_KERNEL_GS_BASE, + // Guest and host access use the same SPEC_CTRL state. + MSR_SPEC_CTRL, + // AMD virtualized SSBD control. Guest-writable only where the host exposes + // the legacy VIRT_SPEC_CTRL SSBD mechanism. Host probing omits it otherwise. + MSR_VIRT_SPEC_CTRL, + // Host probing omits CET state when CET is unavailable. + MSR_U_CET, + MSR_S_CET, + // Host probing omits shadow-stack state when it is unavailable. + MSR_PL0_SSP, + MSR_PL1_SSP, + MSR_PL2_SSP, + MSR_PL3_SSP, + MSR_INTERRUPT_SSP_TABLE_ADDR, + // MSHV maps XSS through its U_XSS alias. + MSR_XSS, + // Guest and host access use the same virtual counter. + MSR_TSC, + // Hyper-V stores TSC_ADJUST independently from TSC. + MSR_TSC_ADJUST, + // Host probing omits TSC_AUX when RDTSCP is unavailable. + MSR_TSC_AUX, + // Hyper-V exposes MPERF and APERF as per-VP counters. + MSR_MPERF, + MSR_APERF, + // Host probing omits TSX_CTRL when TSX control is unavailable. + MSR_TSX_CTRL, + // Host probing omits XFD state when XFD is unavailable. + MSR_XFD, + MSR_XFD_ERR, + // Feature MSRs enabled by default on capable hosts. Host probing omits + // each when its feature is unavailable. + MSR_UMWAIT_CONTROL, + MSR_TSC_DEADLINE, + MSR_BNDCFGS, + // Hyper-V accepts fixed-MTRR writes even when MTRRCAP.FIX is clear. + MSR_MTRR_DEF_TYPE, + 0x200, // PHYSBASE0 + 0x201, // PHYSMASK0 + 0x202, // PHYSBASE1 + 0x203, // PHYSMASK1 + 0x204, // PHYSBASE2 + 0x205, // PHYSMASK2 + 0x206, // PHYSBASE3 + 0x207, // PHYSMASK3 + 0x208, // PHYSBASE4 + 0x209, // PHYSMASK4 + 0x20A, // PHYSBASE5 + 0x20B, // PHYSMASK5 + 0x20C, // PHYSBASE6 + 0x20D, // PHYSMASK6 + 0x20E, // PHYSBASE7 + 0x20F, // PHYSMASK7 + 0x210, // PHYSBASE8 + 0x211, // PHYSMASK8 + 0x212, // PHYSBASE9 + 0x213, // PHYSMASK9 + 0x214, // PHYSBASEA + 0x215, // PHYSMASKA + 0x216, // PHYSBASEB + 0x217, // PHYSMASKB + 0x218, // PHYSBASEC + 0x219, // PHYSMASKC + 0x21A, // PHYSBASED + 0x21B, // PHYSMASKD + 0x21C, // PHYSBASEE + 0x21D, // PHYSMASKE + 0x21E, // PHYSBASEF + 0x21F, // PHYSMASKF + MSR_MTRR_FIX64K_00000, // FIX64K_00000 + 0x258, // FIX16K_80000 + 0x259, // FIX16K_A0000 + 0x268, // FIX4K_C0000 + 0x269, // FIX4K_C8000 + 0x26A, // FIX4K_D0000 + 0x26B, // FIX4K_D8000 + 0x26C, // FIX4K_E0000 + 0x26D, // FIX4K_E8000 + 0x26E, // FIX4K_F0000 + 0x26F, // FIX4K_F8000 +]; + +/// Whether an MSR carries retained state eligible for the reset set. +pub(crate) fn is_resettable_msr(index: u32) -> bool { + MSR_TABLE.contains(&index) +} + +/// Returns core stateful indices for host filtering. +pub(crate) fn core_reset_indices() -> impl Iterator { + MSR_TABLE.iter().copied() +} + +pub(crate) fn hyperv_mtrr_reset_indices( + mtrr_cap: u64, +) -> Result, crate::hypervisor::virtual_machine::CreateVmError> { + use crate::hypervisor::virtual_machine::CreateVmError; + + let advertised = (mtrr_cap & 0xff) as u8; + if advertised > HYPERV_VARIABLE_MTRR_COUNT { + return Err(CreateVmError::UnexpectedVariableMtrrCount { + advertised, + maximum: HYPERV_VARIABLE_MTRR_COUNT, + }); + } + + let mut indices = Vec::with_capacity(1 + usize::from(advertised) * 2 + 11); + indices.push(MSR_MTRR_DEF_TYPE); + indices.extend((0..u32::from(advertised) * 2).map(|offset| 0x200 + offset)); + indices.extend([ + MSR_MTRR_FIX64K_00000, + 0x258, + 0x259, + 0x268, + 0x269, + 0x26A, + 0x26B, + 0x26C, + 0x26D, + 0x26E, + 0x26F, + ]); + Ok(indices) +} + +pub(crate) fn is_mtrr_reset_index(index: u32) -> bool { + index == MSR_MTRR_DEF_TYPE + || (0x200..=0x21F).contains(&index) + || matches!( + index, + MSR_MTRR_FIX64K_00000 + | 0x258 + | 0x259 + | 0x268 + | 0x269 + | 0x26A + | 0x26B + | 0x26C + | 0x26D + | 0x26E + | 0x26F + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::hypervisor::virtual_machine::{CreateVmError, RegisterError}; + + fn state() -> MsrResetState { + MsrResetState { + baseline: vec![ + MsrEntry { + index: MSR_SYSENTER_CS, + value: 0x1C, + }, + MsrEntry { + index: MSR_SYSENTER_ESP, + value: 0x2C, + }, + MsrEntry { + index: MSR_KERNEL_GS_BASE, + value: 0x3C, + }, + ], + allowed: vec![MSR_SYSENTER_CS, MSR_SYSENTER_ESP], + } + } + + #[test] + fn hyperv_mtrr_indices_follow_guest_capability() { + assert_eq!( + hyperv_mtrr_reset_indices(2).unwrap(), + [ + MSR_MTRR_DEF_TYPE, + 0x200, + 0x201, + 0x202, + 0x203, + MSR_MTRR_FIX64K_00000, + 0x258, + 0x259, + 0x268, + 0x269, + 0x26A, + 0x26B, + 0x26C, + 0x26D, + 0x26E, + 0x26F, + ] + ); + let fixed_bit_does_not_change_reset_set = hyperv_mtrr_reset_indices(2 | (1 << 8)).unwrap(); + assert_eq!( + fixed_bit_does_not_change_reset_set, + hyperv_mtrr_reset_indices(2).unwrap() + ); + } + + #[test] + fn hyperv_mtrr_count_rejects_more_than_sixteen_pairs() { + let indices = hyperv_mtrr_reset_indices(16).unwrap(); + assert!(indices.contains(&0x21F)); + assert_eq!(indices.last(), Some(&0x26F)); + assert!(indices.iter().all(|&index| is_resettable_msr(index))); + assert!(matches!( + hyperv_mtrr_reset_indices(17), + Err(CreateVmError::UnexpectedVariableMtrrCount { + advertised: 17, + maximum: 16 + }) + )); + } + + #[test] + fn snapshot_msr_validation_accepts_exact_canonical_set() { + let supplied = vec![ + MsrEntry { + index: MSR_SYSENTER_CS, + value: 1, + }, + MsrEntry { + index: MSR_SYSENTER_ESP, + value: 2, + }, + MsrEntry { + index: MSR_KERNEL_GS_BASE, + value: 3, + }, + ]; + + assert_eq!( + state() + .validate_snapshot(&supplied, &[MSR_SYSENTER_CS]) + .unwrap(), + supplied + ); + } + + #[test] + fn snapshot_msr_validation_baselines_omitted_indices() { + // A snapshot covering a subset of this VM's reset set is accepted. + // The omitted index takes the creation-time baseline value. + let supplied = vec![ + MsrEntry { + index: MSR_SYSENTER_CS, + value: 1, + }, + MsrEntry { + index: MSR_KERNEL_GS_BASE, + value: 3, + }, + ]; + + assert_eq!( + state().validate_snapshot(&supplied, &[]).unwrap(), + vec![ + MsrEntry { + index: MSR_SYSENTER_CS, + value: 1 + }, + MsrEntry { + index: MSR_SYSENTER_ESP, + value: 0x2C + }, + MsrEntry { + index: MSR_KERNEL_GS_BASE, + value: 3 + }, + ] + ); + } + + #[test] + fn snapshot_msr_validation_rejects_non_superset_allow_list() { + // The snapshot allows an MSR the destination does not. + assert!(matches!( + state().validate_snapshot(&[], &[MSR_KERNEL_GS_BASE]), + Err(RegisterError::SnapshotMsrNotAllowed { missing }) if missing == vec![MSR_KERNEL_GS_BASE] + )); + } + + #[test] + fn snapshot_msr_validation_rejects_index_outside_reset_set() { + let supplied = vec![ + MsrEntry { + index: MSR_SYSENTER_CS, + value: 1, + }, + MsrEntry { + index: MSR_SYSENTER_ESP, + value: 2, + }, + MsrEntry { + index: MSR_PAT, + value: 3, + }, + ]; + + assert!(matches!( + state().validate_snapshot(&supplied, &[]), + Err(RegisterError::InvalidSnapshotMsrIndex { index }) if index == MSR_PAT + )); + } + + #[test] + fn snapshot_msr_validation_accepts_empty_canonical_set() { + let state = MsrResetState { + baseline: Vec::new(), + allowed: Vec::new(), + }; + assert!(state.validate_snapshot(&[], &[]).unwrap().is_empty()); + } +} diff --git a/src/hyperlight_host/src/hypervisor/regs/x86_64/special_regs.rs b/src/hyperlight_host/src/hypervisor/regs/x86_64/special_regs.rs index e67ab3b49..adecfb24a 100644 --- a/src/hyperlight_host/src/hypervisor/regs/x86_64/special_regs.rs +++ b/src/hyperlight_host/src/hypervisor/regs/x86_64/special_regs.rs @@ -677,6 +677,16 @@ mod tests { assert_eq!(original, roundtrip); } + /// The guest boots in xAPIC mode. `APIC_BASE` bit 10 (`EXTD`) enables + /// x2APIC, which would make the `0x800`-`0x8FF` MSR range legal. Keeping it + /// clear is why a guest write to an x2APIC MSR faults. + #[test] + fn standard_defaults_leave_x2apic_disabled() { + const X2APIC_ENABLE: u64 = 1 << 10; + let sregs = CommonSpecialRegisters::standard_64bit_defaults(0x1000); + assert_eq!(sregs.apic_base & X2APIC_ENABLE, 0); + } + #[cfg(mshv3)] #[test] fn round_trip_mshv_sregs() { diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs index 3dc8ec87a..87f9f111e 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs @@ -14,16 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ +use std::collections::HashSet; use std::sync::LazyLock; use hyperlight_common::outb::VmAction; #[cfg(gdb)] use kvm_bindings::kvm_guest_debug; use kvm_bindings::{ - kvm_debugregs, kvm_fpu, kvm_regs, kvm_sregs, kvm_userspace_memory_region, kvm_xsave, + Msrs, kvm_debugregs, kvm_enable_cap, kvm_fpu, kvm_msr_entry, kvm_regs, kvm_sregs, + kvm_userspace_memory_region, kvm_xsave, }; use kvm_ioctls::Cap::UserMemory; -use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd}; +use kvm_ioctls::{ + Cap, Kvm, MsrExitReason, MsrFilterDefaultAction, MsrFilterRange, MsrFilterRangeFlags, VcpuExit, + VcpuFd, VmFd, +}; use tracing::{Span, instrument}; #[cfg(feature = "trace_guest")] use tracing_opentelemetry::OpenTelemetrySpanExt; @@ -34,7 +39,7 @@ use vmm_sys_util::eventfd::EventFd; use crate::hypervisor::gdb::{DebugError, DebuggableVm}; use crate::hypervisor::regs::{ CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters, FP_CONTROL_WORD_DEFAULT, - MXCSR_DEFAULT, + MXCSR_DEFAULT, MsrEntry, is_resettable_msr, }; #[cfg(test)] use crate::hypervisor::virtual_machine::XSAVE_BUFFER_SIZE; @@ -110,6 +115,42 @@ pub(crate) struct KvmVm { static KVM: LazyLock> = LazyLock::new(|| Kvm::new().map_err(|e| CreateVmError::HypervisorNotAvailable(e.into()))); +/// Cached host indices reported by `KVM_GET_MSR_INDEX_LIST`. +/// An empty set makes support checks fail closed. +static HOST_MSR_INDICES: LazyLock> = LazyLock::new(|| match KVM.as_ref() { + Ok(kvm) => match kvm.get_msr_index_list() { + Ok(list) => list.as_slice().iter().copied().collect(), + Err(e) => { + tracing::warn!("KVM_GET_MSR_INDEX_LIST failed: {e}"); + HashSet::new() + } + }, + Err(_) => HashSet::new(), +}); + +/// Returns the set of MSR indices the host KVM supports for get/set. +pub(crate) fn host_msr_indices() -> &'static HashSet { + &HOST_MSR_INDICES +} + +/// KVM allows at most this many MSR filter ranges. +const KVM_MSR_FILTER_MAX_RANGES: usize = 16; + +/// Returns the smallest contiguous ranges covering the supplied indices. +fn coalesce_msr_ranges(indices: &[u32]) -> Vec<(u32, usize)> { + let mut sorted: Vec = indices.to_vec(); + sorted.sort_unstable(); + sorted.dedup(); + let mut groups: Vec<(u32, usize)> = Vec::new(); + for idx in sorted { + match groups.last_mut() { + Some((base, count)) if *base + *count as u32 == idx => *count += 1, + _ => groups.push((idx, 1)), + } + } + groups +} + #[cfg(feature = "hw-interrupts")] impl KvmVm { /// Create the in-kernel IRQ chip and register an irqfd for GSI 0. @@ -228,6 +269,20 @@ impl KvmVm { } Ok(VcpuExit::MmioRead(addr, _)) => return Ok(VmExit::MmioRead(addr)), Ok(VcpuExit::MmioWrite(addr, _)) => return Ok(VmExit::MmioWrite(addr)), + // Complete filtered access through the default run path. + Ok(VcpuExit::X86Rdmsr(msr_exit)) => { + let msr_index = msr_exit.index; + *msr_exit.error = 1; + self.complete_filtered_msr_exit()?; + return Ok(VmExit::MsrRead(msr_index)); + } + Ok(VcpuExit::X86Wrmsr(msr_exit)) => { + let msr_index = msr_exit.index; + let value = msr_exit.data; + *msr_exit.error = 1; + self.complete_filtered_msr_exit()?; + return Ok(VmExit::MsrWrite { msr_index, value }); + } #[cfg(gdb)] Ok(VcpuExit::Debug(debug_exit)) => { return Ok(VmExit::Debug { @@ -250,6 +305,23 @@ impl KvmVm { } } + /// Finish a filtered MSR access after its `error` flag was set. + /// Reentering `KVM_RUN` injects `#GP` for the pending access. + /// immediate_exit halts the guest right after, so it never runs past + /// the fault. That reentry returns `EINTR`, which is expected here. + fn complete_filtered_msr_exit(&mut self) -> std::result::Result<(), RunVcpuError> { + self.vcpu_fd.set_kvm_immediate_exit(1); + // `.err()` drops the `Ok(VcpuExit)`, which borrows `vcpu_fd`, keeping + // only the owned error so the borrow ends before the next call. + let err = self.vcpu_fd.run().err(); + self.vcpu_fd.set_kvm_immediate_exit(0); + match err { + None => Ok(()), + Some(e) if e.errno() == libc::EINTR => Ok(()), + Some(e) => Err(RunVcpuError::Unknown(e.into())), + } + } + #[cfg(feature = "hw-interrupts")] fn handle_pv_timer_config(&mut self, data: &[u8]) { use super::super::x86_64::hw_interrupts::handle_pv_timer_config; @@ -275,6 +347,21 @@ impl KvmVm { Ok(VcpuExit::IoOut(port, data)) => Ok(VmExit::IoOut(port, data.to_vec())), Ok(VcpuExit::MmioRead(addr, _)) => Ok(VmExit::MmioRead(addr)), Ok(VcpuExit::MmioWrite(addr, _)) => Ok(VmExit::MmioWrite(addr)), + // Reentering KVM_RUN completes the failed MSR exit and injects #GP. + // immediate_exit prevents further guest execution. + Ok(VcpuExit::X86Rdmsr(msr_exit)) => { + let msr_index = msr_exit.index; + *msr_exit.error = 1; + self.complete_filtered_msr_exit()?; + Ok(VmExit::MsrRead(msr_index)) + } + Ok(VcpuExit::X86Wrmsr(msr_exit)) => { + let msr_index = msr_exit.index; + let value = msr_exit.data; + *msr_exit.error = 1; + self.complete_filtered_msr_exit()?; + Ok(VmExit::MsrWrite { msr_index, value }) + } #[cfg(gdb)] Ok(VcpuExit::Debug(debug_exit)) => Ok(VmExit::Debug { dr6: debug_exit.dr6, @@ -292,6 +379,150 @@ impl KvmVm { ))), } } + + /// Installs a deny filter containing the validated allow list. + /// Requires `KVM_CAP_X86_USER_SPACE_MSR` and `KVM_CAP_X86_MSR_FILTER`. + pub(crate) fn configure_msr_access( + &self, + allowed: &[u32], + ) -> std::result::Result<(), CreateVmError> { + let hv = KVM.as_ref().map_err(|e| e.clone())?; + if !hv.check_extension(Cap::X86UserSpaceMsr) || !hv.check_extension(Cap::X86MsrFilter) { + tracing::error!( + "KVM does not support KVM_CAP_X86_USER_SPACE_MSR or KVM_CAP_X86_MSR_FILTER." + ); + return Err(CreateVmError::MsrFilterNotSupported); + } + + // Every permitted guest write must have restorable host state. + for &msr in allowed { + self.validate_allowed_msr(msr)?; + } + + // Tell KVM to exit to userspace on filtered MSR access. + let cap = kvm_enable_cap { + cap: Cap::X86UserSpaceMsr as u32, + args: [MsrExitReason::Filter.bits() as u64, 0, 0, 0], + ..Default::default() + }; + self.vm_fd + .enable_cap(&cap) + .map_err(|e| CreateVmError::InitializeVm(e.into()))?; + + // Each contiguous group consumes one KVM filter range. + let groups = coalesce_msr_ranges(allowed); + if groups.len() > KVM_MSR_FILTER_MAX_RANGES { + return Err(CreateVmError::TooManyMsrRanges(groups.len())); + } + + // The bitmaps must live through set_msr_filter. + let bitmaps: Vec> = groups + .iter() + .map(|(_, count)| { + let mut bytes = vec![0u8; count.div_ceil(8)]; + for bit in 0..*count { + bytes[bit / 8] |= 1 << (bit % 8); + } + bytes + }) + .collect(); + + // Default deny requires at least one range. + static DENY_BITMAP: [u8; 1] = [0u8]; + let ranges: Vec = if groups.is_empty() { + vec![MsrFilterRange { + flags: MsrFilterRangeFlags::READ | MsrFilterRangeFlags::WRITE, + base: 0, + msr_count: 1, + bitmap: &DENY_BITMAP, + }] + } else { + groups + .iter() + .zip(bitmaps.iter()) + .map(|((base, count), bitmap)| MsrFilterRange { + flags: MsrFilterRangeFlags::READ | MsrFilterRangeFlags::WRITE, + base: *base, + msr_count: *count as u32, + bitmap: bitmap.as_slice(), + }) + .collect() + }; + + self.vm_fd + .set_msr_filter(MsrFilterDefaultAction::DENY, &ranges) + .map_err(|e| CreateVmError::InitializeVm(e.into()))?; + Ok(()) + } + + /// Validates that an allowed MSR has restorable host state. + fn validate_allowed_msr(&self, msr: u32) -> std::result::Result<(), CreateVmError> { + if !is_resettable_msr(msr) { + return Err(CreateVmError::MsrNotAllowable { + msr, + reason: "MSR is not a resettable MSR".to_string(), + }); + } + if !host_msr_indices().contains(&msr) { + return Err(CreateVmError::MsrNotAllowable { + msr, + reason: "MSR is not supported by the host".to_string(), + }); + } + let value = self + .read_msr(msr) + .map_err(|e| CreateVmError::MsrNotAllowable { + msr, + reason: format!("MSR is not readable: {e}"), + })?; + self.write_msr(msr, value) + .map_err(|e| CreateVmError::MsrNotAllowable { + msr, + reason: format!("MSR is not resettable: {e}"), + })?; + Ok(()) + } + + /// Reads one vCPU MSR without the guest filter. + fn read_msr(&self, index: u32) -> std::result::Result { + let mut msrs = Msrs::from_entries(&[kvm_msr_entry { + index, + ..Default::default() + }]) + .map_err(|e| RegisterError::MsrBuild(format!("{e:?}")))?; + let n = self + .vcpu_fd + .get_msrs(&mut msrs) + .map_err(|e| RegisterError::GetMsrs(e.into()))?; + if n != 1 { + return Err(RegisterError::MsrShortCount { + expected: 1, + actual: n, + }); + } + Ok(msrs.as_slice()[0].data) + } + + /// Writes one vCPU MSR without the guest filter. + fn write_msr(&self, index: u32, data: u64) -> std::result::Result<(), RegisterError> { + let msrs = Msrs::from_entries(&[kvm_msr_entry { + index, + data, + ..Default::default() + }]) + .map_err(|e| RegisterError::MsrBuild(format!("{e:?}")))?; + let n = self + .vcpu_fd + .set_msrs(&msrs) + .map_err(|e| RegisterError::SetMsrs(e.into()))?; + if n != 1 { + return Err(RegisterError::MsrShortCount { + expected: 1, + actual: n, + }); + } + Ok(()) + } } impl VirtualMachine for KvmVm { @@ -403,6 +634,66 @@ impl VirtualMachine for KvmVm { Ok(()) } + fn msrs(&self, indices: &[u32]) -> std::result::Result, RegisterError> { + if indices.is_empty() { + return Ok(Vec::new()); + } + let entries: Vec = indices + .iter() + .map(|&index| kvm_msr_entry { + index, + ..Default::default() + }) + .collect(); + let mut msrs = + Msrs::from_entries(&entries).map_err(|e| RegisterError::MsrBuild(format!("{e:?}")))?; + let n = self + .vcpu_fd + .get_msrs(&mut msrs) + .map_err(|e| RegisterError::GetMsrs(e.into()))?; + if n != indices.len() { + return Err(RegisterError::MsrShortCount { + expected: indices.len(), + actual: n, + }); + } + Ok(msrs + .as_slice() + .iter() + .map(|e| MsrEntry { + index: e.index, + value: e.data, + }) + .collect()) + } + + fn set_msrs(&self, msrs: &[MsrEntry]) -> std::result::Result<(), RegisterError> { + let entries: Vec = msrs + .iter() + .map(|e| kvm_msr_entry { + index: e.index, + data: e.value, + ..Default::default() + }) + .collect(); + if entries.is_empty() { + return Ok(()); + } + let kvm_msrs = + Msrs::from_entries(&entries).map_err(|e| RegisterError::MsrBuild(format!("{e:?}")))?; + let n = self + .vcpu_fd + .set_msrs(&kvm_msrs) + .map_err(|e| RegisterError::SetMsrs(e.into()))?; + if n != entries.len() { + return Err(RegisterError::MsrShortCount { + expected: entries.len(), + actual: n, + }); + } + Ok(()) + } + #[allow(dead_code)] fn xsave(&self) -> std::result::Result, RegisterError> { let xsave = self @@ -570,10 +861,45 @@ impl DebuggableVm for KvmVm { } #[cfg(test)] -#[cfg(feature = "hw-interrupts")] -mod hw_interrupt_tests { +mod tests { use super::*; + #[test] + fn coalesces_unsorted_contiguous_indices() { + assert_eq!( + coalesce_msr_ranges(&[0x176, 0x174, 0x175]), + vec![(0x174, 3)] + ); + } + + #[test] + fn deduplicates_indices() { + assert_eq!( + coalesce_msr_ranges(&[0x174, 0x174, 0x176]), + vec![(0x174, 1), (0x176, 1)] + ); + } + + #[test] + fn preserves_sixteen_range_boundary() { + let indices: Vec = (0..KVM_MSR_FILTER_MAX_RANGES as u32) + .map(|index| index * 2) + .collect(); + let ranges = coalesce_msr_ranges(&indices); + assert_eq!(ranges.len(), KVM_MSR_FILTER_MAX_RANGES); + assert!(ranges.iter().all(|(_, count)| *count == 1)); + } + + #[test] + fn scattered_indices_exceed_sixteen_range_limit() { + let indices: Vec = (0..=KVM_MSR_FILTER_MAX_RANGES as u32) + .map(|index| index * 2) + .collect(); + let ranges = coalesce_msr_ranges(&indices); + assert!(ranges.len() > KVM_MSR_FILTER_MAX_RANGES); + } + + #[cfg(feature = "hw-interrupts")] #[test] fn halt_port_is_not_standard_device() { // VmAction::Halt port must not overlap in-kernel PIC/PIT/speaker ports diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs index dac344711..4cd8fca02 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs @@ -21,9 +21,13 @@ use tracing::{Span, instrument}; #[cfg(gdb)] use crate::hypervisor::gdb::DebugError; +#[cfg(target_arch = "x86_64")] +use crate::hypervisor::regs::MsrEntry; use crate::hypervisor::regs::{ CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters, }; +#[cfg(all(target_arch = "x86_64", any(mshv3, target_os = "windows")))] +use crate::hypervisor::regs::{MSR_MTRR_CAP, hyperv_mtrr_reset_indices, is_resettable_msr}; use crate::mem::memory_region::MemoryRegion; #[cfg(feature = "trace_guest")] use crate::sandbox::trace::TraceContext as SandboxTraceContext; @@ -139,6 +143,12 @@ pub(crate) enum VmExit { MmioRead(u64), /// The vCPU tried to write to the given (unmapped) addr MmioWrite(u64), + /// The vCPU tried to read from the given MSR + #[cfg(all(target_arch = "x86_64", kvm))] + MsrRead(u32), + /// The vCPU tried to write to the given MSR with the given value + #[cfg(all(target_arch = "x86_64", kvm))] + MsrWrite { msr_index: u32, value: u64 }, /// The vCPU execution has been cancelled Cancelled(), /// The vCPU has exited for a reason that is not handled by Hyperlight @@ -183,6 +193,30 @@ pub enum CreateVmError { HypervisorNotAvailable(HypervisorError), #[error("Initialize VM failed: {0}")] InitializeVm(HypervisorError), + #[cfg(all(kvm, target_arch = "x86_64"))] + #[error("KVM MSR filtering requires KVM_CAP_X86_USER_SPACE_MSR and KVM_CAP_X86_MSR_FILTER")] + MsrFilterNotSupported, + #[cfg(target_arch = "x86_64")] + #[error("MSR {msr:#x} cannot be allowed: {reason}")] + MsrNotAllowable { msr: u32, reason: String }, + #[cfg(target_arch = "x86_64")] + #[error("Failed to read IA32_MTRRCAP: {0}")] + GetMtrrCap(RegisterError), + #[cfg(target_arch = "x86_64")] + #[error("Guest-visible MTRRs cannot be reset: {0}")] + RequiredMtrrsNotResettable(RegisterError), + #[cfg(target_arch = "x86_64")] + #[error("Guest exposes {advertised} variable MTRR pairs, expected at most {maximum}")] + UnexpectedVariableMtrrCount { advertised: u8, maximum: u8 }, + #[cfg(all(kvm, target_arch = "x86_64"))] + #[error("Too many allowed MSR filter ranges: {0}. Maximum is 16")] + TooManyMsrRanges(usize), + #[cfg(target_os = "windows")] + #[error("Get Partition Property failed: {0}")] + GetPartitionProperty(HypervisorError), + #[cfg(target_os = "windows")] + #[error("WHP exposes {advertised} processor feature banks, expected {expected}")] + UnexpectedProcessorFeatureBankCount { advertised: u32, expected: u32 }, #[error("Set Partition Property failed: {0}")] SetPartitionProperty(HypervisorError), #[cfg(target_os = "windows")] @@ -241,6 +275,38 @@ pub enum RegisterError { }, #[error("Invalid xsave alignment")] InvalidXsaveAlignment, + #[cfg(target_arch = "x86_64")] + #[error("MSR operation not supported on this hypervisor")] + MsrsUnsupported, + #[cfg(target_arch = "x86_64")] + #[error("Failed to build MSR list: {0}")] + MsrBuild(String), + #[cfg(target_arch = "x86_64")] + #[error("Failed to get MSRs: {0}")] + GetMsrs(HypervisorError), + #[cfg(target_arch = "x86_64")] + #[error("Failed to set MSRs: {0}")] + SetMsrs(HypervisorError), + #[cfg(target_arch = "x86_64")] + #[error("Snapshot allows MSRs the destination does not: {missing:x?}")] + SnapshotMsrNotAllowed { + /// Allowed MSR indices present in the snapshot but not the destination. + missing: Vec, + }, + #[cfg(target_arch = "x86_64")] + #[error("Snapshot MSR index {index:#x} is not in this VM's reset set")] + InvalidSnapshotMsrIndex { + /// Architectural MSR index supplied by the snapshot. + index: u32, + }, + #[cfg(all(kvm, target_arch = "x86_64"))] + #[error("MSR batch short count: expected {expected}, applied {actual}")] + MsrShortCount { + /// Number of MSRs requested + expected: usize, + /// Number of MSRs actually applied before KVM stopped + actual: usize, + }, #[cfg(target_os = "windows")] #[error("Failed to get xsave size: {0}")] GetXsaveSize(#[from] HypervisorError), @@ -359,6 +425,13 @@ pub(crate) trait VirtualMachine: Debug + Send { #[allow(dead_code)] fn set_debug_regs(&self, drs: &CommonDebugRegs) -> std::result::Result<(), RegisterError>; + /// Reads the requested MSRs. + #[cfg(target_arch = "x86_64")] + fn msrs(&self, indices: &[u32]) -> std::result::Result, RegisterError>; + /// Writes the supplied MSRs. + #[cfg(target_arch = "x86_64")] + fn set_msrs(&self, msrs: &[MsrEntry]) -> std::result::Result<(), RegisterError>; + /// Get xsave #[allow(dead_code)] #[cfg(not(target_arch = "aarch64"))] @@ -385,6 +458,51 @@ pub(crate) trait VirtualMachine: Debug + Send { fn partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE; } +/// Validates that each allowed MSR is restorable on a filterless (MSHV/WHP) +/// host: reset replays a captured value, so the host must read and write it. +/// Rejects e.g. a variable MTRR above the host VCNT. +#[cfg(all(target_arch = "x86_64", any(mshv3, target_os = "windows")))] +pub(crate) fn validate_allowed_msrs( + vm: &dyn VirtualMachine, + allowed: &[u32], +) -> std::result::Result<(), CreateVmError> { + for &msr in allowed { + if !is_resettable_msr(msr) { + return Err(CreateVmError::MsrNotAllowable { + msr, + reason: "MSR is not a resettable MSR".to_string(), + }); + } + let captured = vm + .msrs(&[msr]) + .map_err(|_| CreateVmError::MsrNotAllowable { + msr, + reason: "MSR cannot be read on this host".to_string(), + })?; + vm.set_msrs(&captured) + .map_err(|_| CreateVmError::MsrNotAllowable { + msr, + reason: "MSR cannot be written on this host".to_string(), + })?; + } + Ok(()) +} + +/// Returns every guest-visible MTRR a filterless (MSHV/WHP) host must reset. +#[cfg(all(target_arch = "x86_64", any(mshv3, target_os = "windows")))] +pub(crate) fn mtrr_reset_indices( + vm: &dyn VirtualMachine, +) -> std::result::Result, CreateVmError> { + let mtrr_cap = vm + .msrs(&[MSR_MTRR_CAP]) + .map_err(CreateVmError::GetMtrrCap)?[0] + .value; + let indices = hyperv_mtrr_reset_indices(mtrr_cap)?; + vm.msrs(&indices) + .map_err(CreateVmError::RequiredMtrrsNotResettable)?; + Ok(indices) +} + #[cfg(test)] mod tests { diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs index 1fd50d29a..88f2ae1f3 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs @@ -31,8 +31,10 @@ use mshv_bindings::{ hv_message_type_HVMSG_X64_HALT, hv_message_type_HVMSG_X64_IO_PORT_INTERCEPT, hv_partition_property_code_HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES, hv_partition_synthetic_processor_features, hv_register_assoc, - hv_register_name_HV_X64_REGISTER_RIP, hv_register_value, mshv_create_partition_v2, - mshv_user_mem_region, + hv_register_name_HV_X64_REGISTER_MSR_MTRR_PHYS_BASE0, + hv_register_name_HV_X64_REGISTER_MSR_MTRR_PHYS_MASK0, hv_register_name_HV_X64_REGISTER_RIP, + hv_register_name_HV_X64_REGISTER_U_XSS, hv_register_value, mshv_create_partition_v2, + mshv_user_mem_region, msr_to_hv_reg_name as mshv_msr_to_hv_reg_name, }; #[cfg(feature = "hw-interrupts")] use mshv_bindings::{ @@ -50,7 +52,8 @@ use tracing_opentelemetry::OpenTelemetrySpanExt; use crate::hypervisor::gdb::{DebugError, DebuggableVm}; use crate::hypervisor::regs::{ CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters, FP_CONTROL_WORD_DEFAULT, - MXCSR_DEFAULT, + MSR_APERF, MSR_BNDCFGS, MSR_MPERF, MSR_TSC_DEADLINE, MSR_TSX_CTRL, MSR_UMWAIT_CONTROL, + MSR_VIRT_SPEC_CTRL, MSR_XFD, MSR_XFD_ERR, MSR_XSS, MXCSR_DEFAULT, MsrEntry, }; #[cfg(test)] use crate::hypervisor::virtual_machine::XSAVE_BUFFER_SIZE; @@ -77,6 +80,84 @@ pub(crate) fn is_hypervisor_present() -> bool { } } +/// Maps an MSR index to the Hyper-V register used for host reset. +fn msr_to_hv_register_name(index: u32) -> Result { + const HV_X64_REGISTER_BNDCFGS: u32 = 0x0008_007C; + const HV_X64_REGISTER_MCOUNT: u32 = 0x0008_007E; + const HV_X64_REGISTER_A_COUNT: u32 = 0x0008_007F; + const HV_X64_REGISTER_TSX_CTRL: u32 = 0x0008_0088; + const HV_X64_REGISTER_TSC_DEADLINE: u32 = 0x0008_0095; + const HV_X64_REGISTER_UMWAIT_CONTROL: u32 = 0x0008_0098; + const HV_X64_REGISTER_XFD: u32 = 0x0008_0099; + const HV_X64_REGISTER_XFD_ERR: u32 = 0x0008_009A; + const HV_X64_REGISTER_VIRT_SPEC_CTRL: u32 = 0x0008_0086; + + if (0x200..=0x21F).contains(&index) { + let pair = (index - 0x200) / 2; + return Ok(if index & 1 == 0 { + hv_register_name_HV_X64_REGISTER_MSR_MTRR_PHYS_BASE0 + pair + } else { + hv_register_name_HV_X64_REGISTER_MSR_MTRR_PHYS_MASK0 + pair + }); + } + if index == MSR_XSS { + return Ok(hv_register_name_HV_X64_REGISTER_U_XSS); + } + if index == MSR_MPERF { + return Ok(HV_X64_REGISTER_MCOUNT); + } + if index == MSR_APERF { + return Ok(HV_X64_REGISTER_A_COUNT); + } + if index == MSR_TSX_CTRL { + return Ok(HV_X64_REGISTER_TSX_CTRL); + } + if index == MSR_TSC_DEADLINE { + return Ok(HV_X64_REGISTER_TSC_DEADLINE); + } + if index == MSR_UMWAIT_CONTROL { + return Ok(HV_X64_REGISTER_UMWAIT_CONTROL); + } + if index == MSR_BNDCFGS { + return Ok(HV_X64_REGISTER_BNDCFGS); + } + if index == MSR_XFD { + return Ok(HV_X64_REGISTER_XFD); + } + if index == MSR_XFD_ERR { + return Ok(HV_X64_REGISTER_XFD_ERR); + } + if index == MSR_VIRT_SPEC_CTRL { + return Ok(HV_X64_REGISTER_VIRT_SPEC_CTRL); + } + mshv_msr_to_hv_reg_name(index) +} + +#[cfg(test)] +mod msr_mapping_tests { + use super::*; + use crate::hypervisor::regs::core_reset_indices; + + #[test] + fn maps_all_stateful_msrs() { + for index in core_reset_indices() { + assert!( + msr_to_hv_register_name(index).is_ok(), + "missing MSR mapping for {index:#x}" + ); + } + assert_eq!(msr_to_hv_register_name(MSR_MPERF), Ok(0x0008_007E)); + assert_eq!(msr_to_hv_register_name(MSR_APERF), Ok(0x0008_007F)); + assert_eq!(msr_to_hv_register_name(MSR_TSX_CTRL), Ok(0x0008_0088)); + assert_eq!(msr_to_hv_register_name(MSR_TSC_DEADLINE), Ok(0x0008_0095)); + assert_eq!(msr_to_hv_register_name(MSR_UMWAIT_CONTROL), Ok(0x0008_0098)); + assert_eq!(msr_to_hv_register_name(MSR_BNDCFGS), Ok(0x0008_007C)); + assert_eq!(msr_to_hv_register_name(MSR_XFD), Ok(0x0008_0099)); + assert_eq!(msr_to_hv_register_name(MSR_XFD_ERR), Ok(0x0008_009A)); + assert_eq!(msr_to_hv_register_name(MSR_VIRT_SPEC_CTRL), Ok(0x0008_0086)); + } +} + /// A MSHV implementation of a single-vcpu VM #[derive(Debug)] pub(crate) struct MshvVm { @@ -102,13 +183,13 @@ impl MshvVm { let mshv = MSHV.as_ref().map_err(|e| e.clone())?; #[allow(unused_mut)] - let mut pr: mshv_create_partition_v2 = Default::default(); + let mut pr = mshv_create_partition_v2::default(); // Enable LAPIC for hw-interrupts — required for interrupt delivery // via request_virtual_interrupt. #[cfg(feature = "hw-interrupts")] { use mshv_bindings::MSHV_PT_BIT_LAPIC; - pr.pt_flags = 1u64 << MSHV_PT_BIT_LAPIC; + pr.pt_flags |= 1u64 << MSHV_PT_BIT_LAPIC; } // It's important to use create_vm_with_args() (not create_vm()), // because create_vm() sets up a SynIC partition by default. @@ -432,6 +513,55 @@ impl VirtualMachine for MshvVm { Ok(()) } + fn msrs(&self, indices: &[u32]) -> std::result::Result, RegisterError> { + if indices.is_empty() { + return Ok(Vec::new()); + } + let mut registers: Vec = indices + .iter() + .map(|&index| { + Ok(hv_register_assoc { + name: msr_to_hv_register_name(index) + .map_err(|_| RegisterError::MsrsUnsupported)?, + ..Default::default() + }) + }) + .collect::>()?; + self.vcpu_fd + .get_reg(&mut registers) + .map_err(|e| RegisterError::GetMsrs(e.into()))?; + Ok(registers + .iter() + .zip(indices) + .map(|(register, &index)| MsrEntry { + index, + // SAFETY: get_reg initialized each association as a 64-bit register. + value: unsafe { register.value.reg64 }, + }) + .collect()) + } + + fn set_msrs(&self, msrs: &[MsrEntry]) -> std::result::Result<(), RegisterError> { + let registers: Vec = msrs + .iter() + .map(|entry| { + Ok(hv_register_assoc { + name: msr_to_hv_register_name(entry.index) + .map_err(|_| RegisterError::MsrsUnsupported)?, + value: hv_register_value { reg64: entry.value }, + ..Default::default() + }) + }) + .collect::>()?; + if registers.is_empty() { + return Ok(()); + } + self.vcpu_fd + .set_reg(®isters) + .map_err(|e| RegisterError::SetMsrs(e.into()))?; + Ok(()) + } + #[allow(dead_code)] fn xsave(&self) -> std::result::Result, RegisterError> { let xsave = self diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs index 6de2b29f1..7f3ab0656 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs @@ -33,9 +33,13 @@ use windows_result::HRESULT; use crate::hypervisor::gdb::{DebugError, DebuggableVm}; use crate::hypervisor::regs::{ Align16, CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters, - FP_CONTROL_WORD_DEFAULT, MXCSR_DEFAULT, WHP_DEBUG_REGS_NAMES, WHP_DEBUG_REGS_NAMES_LEN, - WHP_FPU_NAMES, WHP_FPU_NAMES_LEN, WHP_REGS_NAMES, WHP_REGS_NAMES_LEN, WHP_SREGS_NAMES, - WHP_SREGS_NAMES_LEN, + FP_CONTROL_WORD_DEFAULT, MSR_APERF, MSR_BNDCFGS, MSR_CSTAR, MSR_INTERRUPT_SSP_TABLE_ADDR, + MSR_KERNEL_GS_BASE, MSR_LSTAR, MSR_MPERF, MSR_MTRR_CAP, MSR_PAT, MSR_PL0_SSP, MSR_PL1_SSP, + MSR_PL2_SSP, MSR_PL3_SSP, MSR_S_CET, MSR_SFMASK, MSR_SPEC_CTRL, MSR_STAR, MSR_SYSENTER_CS, + MSR_SYSENTER_EIP, MSR_SYSENTER_ESP, MSR_TSC_DEADLINE, MSR_TSX_CTRL, MSR_U_CET, + MSR_UMWAIT_CONTROL, MSR_XFD, MSR_XFD_ERR, MXCSR_DEFAULT, MsrEntry, WHP_DEBUG_REGS_NAMES, + WHP_DEBUG_REGS_NAMES_LEN, WHP_FPU_NAMES, WHP_FPU_NAMES_LEN, WHP_REGS_NAMES, WHP_REGS_NAMES_LEN, + WHP_SREGS_NAMES, WHP_SREGS_NAMES_LEN, }; use crate::hypervisor::surrogate_process::SurrogateProcess; use crate::hypervisor::surrogate_process_manager::{ @@ -73,6 +77,143 @@ pub(crate) fn is_hypervisor_present() -> bool { } } +/// Maps an MSR index to the WHP register used for host reset. +fn msr_to_whv_register_name(index: u32) -> Option { + Some(match index { + MSR_MTRR_CAP => WHvX64RegisterMsrMtrrCap, + MSR_SYSENTER_CS => WHvX64RegisterSysenterCs, + MSR_SYSENTER_ESP => WHvX64RegisterSysenterEsp, + MSR_SYSENTER_EIP => WHvX64RegisterSysenterEip, + MSR_PAT => WHvX64RegisterPat, + MSR_STAR => WHvX64RegisterStar, + MSR_LSTAR => WHvX64RegisterLstar, + MSR_CSTAR => WHvX64RegisterCstar, + MSR_SFMASK => WHvX64RegisterSfmask, + MSR_KERNEL_GS_BASE => WHvX64RegisterKernelGsBase, + MSR_SPEC_CTRL => WHvX64RegisterSpecCtrl, + MSR_U_CET => WHvX64RegisterUCet, + MSR_S_CET => WHvX64RegisterSCet, + MSR_PL0_SSP => WHvX64RegisterPl0Ssp, + MSR_PL1_SSP => WHvX64RegisterPl1Ssp, + MSR_PL2_SSP => WHvX64RegisterPl2Ssp, + MSR_PL3_SSP => WHvX64RegisterPl3Ssp, + MSR_INTERRUPT_SSP_TABLE_ADDR => WHvX64RegisterInterruptSspTableAddr, + // TSC and TSC offset/aux. + 0x10 => WHvX64RegisterTsc, + 0x3B => WHvX64RegisterTscAdjust, + 0xC000_0103 => WHvX64RegisterTscAux, + MSR_MPERF => WHvX64RegisterMCount, + MSR_APERF => WHvX64RegisterACount, + MSR_TSX_CTRL => WHvX64RegisterTsxCtrl, + MSR_XFD => WHvX64RegisterXfd, + MSR_XFD_ERR => WHvX64RegisterXfdErr, + MSR_UMWAIT_CONTROL => WHvX64RegisterUmwaitControl, + MSR_TSC_DEADLINE => WHvX64RegisterTscDeadline, + MSR_BNDCFGS => WHvX64RegisterBndcfgs, + // XSAVE supervisor state mask (IA32_XSS). + 0xDA0 => WHvX64RegisterXss, + // MTRRs: def type, variable base/mask pairs 0..=9, fixed ranges. + 0x2FF => WHvX64RegisterMsrMtrrDefType, + 0x200 => WHvX64RegisterMsrMtrrPhysBase0, + 0x201 => WHvX64RegisterMsrMtrrPhysMask0, + 0x202 => WHvX64RegisterMsrMtrrPhysBase1, + 0x203 => WHvX64RegisterMsrMtrrPhysMask1, + 0x204 => WHvX64RegisterMsrMtrrPhysBase2, + 0x205 => WHvX64RegisterMsrMtrrPhysMask2, + 0x206 => WHvX64RegisterMsrMtrrPhysBase3, + 0x207 => WHvX64RegisterMsrMtrrPhysMask3, + 0x208 => WHvX64RegisterMsrMtrrPhysBase4, + 0x209 => WHvX64RegisterMsrMtrrPhysMask4, + 0x20A => WHvX64RegisterMsrMtrrPhysBase5, + 0x20B => WHvX64RegisterMsrMtrrPhysMask5, + 0x20C => WHvX64RegisterMsrMtrrPhysBase6, + 0x20D => WHvX64RegisterMsrMtrrPhysMask6, + 0x20E => WHvX64RegisterMsrMtrrPhysBase7, + 0x20F => WHvX64RegisterMsrMtrrPhysMask7, + 0x210 => WHvX64RegisterMsrMtrrPhysBase8, + 0x211 => WHvX64RegisterMsrMtrrPhysMask8, + 0x212 => WHvX64RegisterMsrMtrrPhysBase9, + 0x213 => WHvX64RegisterMsrMtrrPhysMask9, + 0x214 => WHvX64RegisterMsrMtrrPhysBaseA, + 0x215 => WHvX64RegisterMsrMtrrPhysMaskA, + 0x216 => WHvX64RegisterMsrMtrrPhysBaseB, + 0x217 => WHvX64RegisterMsrMtrrPhysMaskB, + 0x218 => WHvX64RegisterMsrMtrrPhysBaseC, + 0x219 => WHvX64RegisterMsrMtrrPhysMaskC, + 0x21A => WHvX64RegisterMsrMtrrPhysBaseD, + 0x21B => WHvX64RegisterMsrMtrrPhysMaskD, + 0x21C => WHvX64RegisterMsrMtrrPhysBaseE, + 0x21D => WHvX64RegisterMsrMtrrPhysMaskE, + 0x21E => WHvX64RegisterMsrMtrrPhysBaseF, + 0x21F => WHvX64RegisterMsrMtrrPhysMaskF, + 0x250 => WHvX64RegisterMsrMtrrFix64k00000, + 0x258 => WHvX64RegisterMsrMtrrFix16k80000, + 0x259 => WHvX64RegisterMsrMtrrFix16kA0000, + 0x268 => WHvX64RegisterMsrMtrrFix4kC0000, + 0x269 => WHvX64RegisterMsrMtrrFix4kC8000, + 0x26A => WHvX64RegisterMsrMtrrFix4kD0000, + 0x26B => WHvX64RegisterMsrMtrrFix4kD8000, + 0x26C => WHvX64RegisterMsrMtrrFix4kE0000, + 0x26D => WHvX64RegisterMsrMtrrFix4kE8000, + 0x26E => WHvX64RegisterMsrMtrrFix4kF0000, + 0x26F => WHvX64RegisterMsrMtrrFix4kF8000, + _ => return None, + }) +} + +#[cfg(test)] +mod msr_mapping_tests { + use super::*; + use crate::hypervisor::regs::{MSR_DEBUGCTL, MSR_VIRT_SPEC_CTRL, core_reset_indices}; + + #[test] + fn maps_all_stateful_msrs_except_debugctl_and_virt_spec_ctrl() { + for index in core_reset_indices() { + // WHP models neither DEBUGCTL nor VIRT_SPEC_CTRL as a named register. + // Both are safe: WHP does not expose their guest-writable features, so + // a guest cannot leave retained state in them. + if index != MSR_DEBUGCTL && index != MSR_VIRT_SPEC_CTRL { + assert!( + msr_to_whv_register_name(index).is_some(), + "missing MSR mapping for {index:#x}" + ); + } + } + + assert!(msr_to_whv_register_name(MSR_DEBUGCTL).is_none()); + assert!(msr_to_whv_register_name(MSR_VIRT_SPEC_CTRL).is_none()); + assert_eq!( + msr_to_whv_register_name(MSR_MPERF), + Some(WHvX64RegisterMCount) + ); + assert_eq!( + msr_to_whv_register_name(MSR_APERF), + Some(WHvX64RegisterACount) + ); + assert_eq!( + msr_to_whv_register_name(MSR_TSX_CTRL), + Some(WHvX64RegisterTsxCtrl) + ); + assert_eq!(msr_to_whv_register_name(MSR_XFD), Some(WHvX64RegisterXfd)); + assert_eq!( + msr_to_whv_register_name(MSR_XFD_ERR), + Some(WHvX64RegisterXfdErr) + ); + assert_eq!( + msr_to_whv_register_name(MSR_UMWAIT_CONTROL), + Some(WHvX64RegisterUmwaitControl) + ); + assert_eq!( + msr_to_whv_register_name(MSR_TSC_DEADLINE), + Some(WHvX64RegisterTscDeadline) + ); + assert_eq!( + msr_to_whv_register_name(MSR_BNDCFGS), + Some(WHvX64RegisterBndcfgs) + ); + } +} + /// Helper: release a host-side file mapping view and its handle. /// Called from both `unmap_memory` and `WhpVm::drop`. fn release_file_mapping(view_base: *mut c_void, mapping_handle: HandleWrapper) { @@ -177,6 +318,9 @@ impl WhpVm { #[cfg(feature = "hw-interrupts")] Self::enable_lapic_emulation(p)?; + // Hyper-V permits MSR intercepts only for unimplemented indices. + // Implemented MSR isolation therefore depends on reset. + WHvSetupPartition(p).map_err(|e| CreateVmError::InitializeVm(e.into()))?; WHvCreateVirtualProcessor(p, 0, 0) .map_err(|e| CreateVmError::CreateVcpuFd(e.into()))?; @@ -667,6 +811,59 @@ impl VirtualMachine for WhpVm { } } + fn msrs(&self, indices: &[u32]) -> std::result::Result, RegisterError> { + if indices.is_empty() { + return Ok(Vec::new()); + } + // Callers validate every index before reaching this mapping. + let names: Vec = indices + .iter() + .map(|&i| msr_to_whv_register_name(i).ok_or(RegisterError::MsrsUnsupported)) + .collect::>()?; + // SAFETY: WHV_REGISTER_VALUE is a union of plain-old-data fields, so an + // all-zero value is a valid initial state. + let mut values: Vec> = + vec![unsafe { std::mem::zeroed() }; names.len()]; + // SAFETY: names and values have equal length. The call fills each value + // slot from the partition's vp 0 for the given register names. + unsafe { + WHvGetVirtualProcessorRegisters( + self.partition, + 0, + names.as_ptr(), + names.len() as u32, + values.as_mut_ptr() as *mut WHV_REGISTER_VALUE, + ) + .map_err(|e| RegisterError::GetMsrs(e.into()))?; + } + Ok(indices + .iter() + .zip(values) + .map(|(&index, v)| MsrEntry { + index, + // SAFETY: each register was read as a 64-bit MSR value. + value: unsafe { v.0.Reg64 }, + }) + .collect()) + } + + fn set_msrs(&self, msrs: &[MsrEntry]) -> std::result::Result<(), RegisterError> { + let regs: Vec<(WHV_REGISTER_NAME, Align16)> = msrs + .iter() + .map(|e| { + msr_to_whv_register_name(e.index) + .map(|name| (name, Align16(WHV_REGISTER_VALUE { Reg64: e.value }))) + .ok_or(RegisterError::MsrsUnsupported) + }) + .collect::>()?; + if regs.is_empty() { + return Ok(()); + } + self.set_registers(®s) + .map_err(|e| RegisterError::SetMsrs(e.into()))?; + Ok(()) + } + fn debug_regs(&self) -> std::result::Result { let mut whp_debug_regs_values: [Align16; WHP_DEBUG_REGS_NAMES_LEN] = Default::default(); diff --git a/src/hyperlight_host/src/mem/mgr.rs b/src/hyperlight_host/src/mem/mgr.rs index 1804d1c9f..57851f215 100644 --- a/src/hyperlight_host/src/mem/mgr.rs +++ b/src/hyperlight_host/src/mem/mgr.rs @@ -31,6 +31,8 @@ use super::shared_mem::{ ExclusiveSharedMemory, GuestSharedMemory, HostSharedMemory, ReadonlySharedMemory, SharedMemory, }; use crate::hypervisor::regs::CommonSpecialRegisters; +#[cfg(target_arch = "x86_64")] +use crate::hypervisor::regs::MsrEntry; use crate::mem::memory_region::MemoryRegion; #[cfg(crashdump)] use crate::mem::memory_region::{CrashDumpRegion, MemoryRegionFlags, MemoryRegionType}; @@ -300,6 +302,8 @@ where root_pt_gpas: &[u64], rsp_gva: u64, sregs: CommonSpecialRegisters, + #[cfg(target_arch = "x86_64")] msrs: Option>, + #[cfg(target_arch = "x86_64")] allowed_msrs: Option>, entrypoint: NextAction, host_functions: HostFunctionDetails, ) -> Result { @@ -313,6 +317,10 @@ where root_pt_gpas, rsp_gva, sregs, + #[cfg(target_arch = "x86_64")] + msrs, + #[cfg(target_arch = "x86_64")] + allowed_msrs, entrypoint, self.snapshot_count, host_functions, diff --git a/src/hyperlight_host/src/sandbox/config.rs b/src/hyperlight_host/src/sandbox/config.rs index f12387a0b..ba5fffdb2 100644 --- a/src/hyperlight_host/src/sandbox/config.rs +++ b/src/hyperlight_host/src/sandbox/config.rs @@ -29,6 +29,18 @@ pub struct DebugInfo { pub port: u16, } +/// Errors returned when configuring guest MSR access. +#[cfg(target_arch = "x86_64")] +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub enum AllowMsrError { + /// The requested allow list exceeds its fixed capacity. + #[error("MSR allow list exceeds its maximum of {maximum} distinct entries")] + CapacityExceeded { + /// Maximum number of distinct allowed MSRs. + maximum: usize, + }, +} + /// The complete set of configuration needed to create a Sandbox #[derive(Copy, Clone, Debug, Eq, PartialEq)] #[repr(C)] @@ -74,6 +86,12 @@ pub struct SandboxConfiguration { interrupt_vcpu_sigrtmin_offset: u8, /// How much writable memory to offer the guest scratch_size: usize, + /// Requested allow list stored inline to keep this type `Copy`. + #[cfg(target_arch = "x86_64")] + allowed_msrs: [u32; Self::MAX_ALLOWED_MSRS], + /// Number of valid entries in `allowed_msrs`. + #[cfg(target_arch = "x86_64")] + allowed_msrs_count: usize, } impl SandboxConfiguration { @@ -93,6 +111,11 @@ impl SandboxConfiguration { pub const DEFAULT_HEAP_SIZE: u64 = 131072; /// The default size of the scratch region pub const DEFAULT_SCRATCH_SIZE: usize = 0x48000; + /// Maximum number of distinct requested MSRs. + /// KVM supports at most 16 MSR filter ranges. Each index may require its + /// own range, so 16 is the portable limit across backends. + #[cfg(target_arch = "x86_64")] + pub const MAX_ALLOWED_MSRS: usize = 16; #[allow(clippy::too_many_arguments)] /// Create a new configuration for a sandbox with the given sizes. @@ -118,6 +141,10 @@ impl SandboxConfiguration { guest_debug_info, #[cfg(crashdump)] guest_core_dump, + #[cfg(target_arch = "x86_64")] + allowed_msrs: [0; Self::MAX_ALLOWED_MSRS], + #[cfg(target_arch = "x86_64")] + allowed_msrs_count: 0, } } @@ -159,6 +186,49 @@ impl SandboxConfiguration { self.interrupt_vcpu_sigrtmin_offset } + /// Adds MSRs to the sandbox allow list. + /// VM creation verifies that the backend can restore them. + /// + /// Duplicate indices, within the slice or against the existing list, are + /// ignored and do not count toward capacity. + /// + /// # Errors + /// + /// Returns [`AllowMsrError::CapacityExceeded`] if the distinct entries + /// would exceed [`Self::MAX_ALLOWED_MSRS`]. The allow list is unchanged on + /// error. + #[cfg(target_arch = "x86_64")] + #[instrument(skip_all, parent = Span::current(), level= "Trace")] + pub fn allow_msrs(&mut self, indices: &[u32]) -> Result<&mut Self, AllowMsrError> { + let additional = indices + .iter() + .enumerate() + .filter(|(position, index)| { + !self.allowed_msrs[..self.allowed_msrs_count].contains(index) + && !indices[..*position].contains(index) + }) + .count(); + if additional > Self::MAX_ALLOWED_MSRS - self.allowed_msrs_count { + return Err(AllowMsrError::CapacityExceeded { + maximum: Self::MAX_ALLOWED_MSRS, + }); + } + for &index in indices { + if !self.allowed_msrs[..self.allowed_msrs_count].contains(&index) { + self.allowed_msrs[self.allowed_msrs_count] = index; + self.allowed_msrs_count += 1; + } + } + Ok(self) + } + + /// Returns the requested allow list. + #[cfg(target_arch = "x86_64")] + #[instrument(skip_all, parent = Span::current(), level= "Trace")] + pub(crate) fn get_allowed_msrs(&self) -> &[u32] { + &self.allowed_msrs[..self.allowed_msrs_count] + } + /// Sets the offset from `SIGRTMIN` to determine the real-time signal used for /// interrupting the VCPU thread. /// @@ -261,7 +331,63 @@ impl Default for SandboxConfiguration { #[cfg(test)] mod tests { - use super::SandboxConfiguration; + use super::{AllowMsrError, SandboxConfiguration}; + + #[test] + #[cfg(target_arch = "x86_64")] + fn msr_allow_list_reports_overflow() { + let mut cfg = SandboxConfiguration::default(); + for index in 0..SandboxConfiguration::MAX_ALLOWED_MSRS as u32 { + cfg.allow_msrs(&[index]).unwrap(); + } + + cfg.allow_msrs(&[0]).unwrap(); + assert_eq!( + cfg.allow_msrs(&[SandboxConfiguration::MAX_ALLOWED_MSRS as u32]), + Err(AllowMsrError::CapacityExceeded { + maximum: SandboxConfiguration::MAX_ALLOWED_MSRS, + }) + ); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn bulk_msr_allow_list_overflow_is_atomic() { + let mut cfg = SandboxConfiguration::default(); + cfg.allow_msrs(&[1, 2]).unwrap(); + let oversized: Vec = (3..=SandboxConfiguration::MAX_ALLOWED_MSRS as u32 + 1).collect(); + + assert!(matches!( + cfg.allow_msrs(&oversized), + Err(AllowMsrError::CapacityExceeded { .. }) + )); + assert_eq!(cfg.get_allowed_msrs(), &[1, 2]); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn allow_msrs_dedups_and_preserves_order() { + let mut cfg = SandboxConfiguration::default(); + cfg.allow_msrs(&[0x10]).unwrap(); + cfg.allow_msrs(&[0x20, 0x20, 0x10, 0x30, 0x20]).unwrap(); + // 0x10 already present, 0x20 and 0x30 added once each in first-seen order. + assert_eq!(cfg.get_allowed_msrs(), &[0x10, 0x20, 0x30]); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn allow_msrs_duplicates_do_not_count_toward_capacity() { + let mut cfg = SandboxConfiguration::default(); + let fill: Vec = (0..SandboxConfiguration::MAX_ALLOWED_MSRS as u32 - 1).collect(); + cfg.allow_msrs(&fill).unwrap(); + // One slot remains. Three copies of one new index count as a single + // distinct entry and fit. + cfg.allow_msrs(&[u32::MAX, u32::MAX, u32::MAX]).unwrap(); + assert_eq!( + cfg.get_allowed_msrs().len(), + SandboxConfiguration::MAX_ALLOWED_MSRS + ); + } #[test] fn overrides() { diff --git a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs index 6f9e87e17..e4dfb155d 100644 --- a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs +++ b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs @@ -159,7 +159,9 @@ impl MultiUseSandbox { /// interrupt behavior. Memory layout fields /// (`input_data_size`, `output_data_size`, `heap_size`, `scratch_size`) /// are always taken from the snapshot. Any values supplied in - /// `config` for those fields are ignored. + /// `config` for those fields are ignored. On x86_64 the `config` MSR + /// allow list must be a superset of the one the snapshot was taken with, + /// or the load fails with an MSR mismatch. /// /// # Examples /// @@ -304,6 +306,15 @@ impl MultiUseSandbox { crate::hypervisor::hyperlight_vm::HyperlightVmError::Restore(e.into()), ) })?; + + // Restore captured MSR state. + #[cfg(target_arch = "x86_64")] + vm.restore_msrs(snapshot.msrs(), snapshot.allowed_msrs()) + .map_err(|e| { + crate::HyperlightError::HyperlightVmError( + crate::hypervisor::hyperlight_vm::HyperlightVmError::Restore(e), + ) + })?; } #[cfg(gdb)] @@ -385,6 +396,13 @@ impl MultiUseSandbox { .vm .get_snapshot_sregs() .map_err(|e| HyperlightError::HyperlightVmError(e.into()))?; + #[cfg(target_arch = "x86_64")] + let msrs = self + .vm + .get_msr_reset_state() + .map_err(|e| HyperlightError::HyperlightVmError(e.into()))?; + #[cfg(target_arch = "x86_64")] + let allowed_msrs = self.vm.get_msr_allow_list(); let entrypoint = self.vm.get_entrypoint(); let host_functions = (&*self.host_funcs.try_lock().map_err(|e| { crate::new_error!("Error locking host_funcs at {}:{}: {}", file!(), line!(), e) @@ -396,6 +414,10 @@ impl MultiUseSandbox { &root_pt_gpas, stack_top_gpa, sregs, + #[cfg(target_arch = "x86_64")] + Some(msrs), + #[cfg(target_arch = "x86_64")] + Some(allowed_msrs), entrypoint, host_functions, )?; @@ -417,6 +439,10 @@ impl MultiUseSandbox { /// [`SnapshotHostFunctionMismatch`](crate::HyperlightError::SnapshotHostFunctionMismatch) /// carrying the missing names and signature differences. /// + /// On x86_64 this sandbox's MSR allow list must be a superset of the one + /// the snapshot was taken with, or the restore poisons with an MSR + /// mismatch. + /// /// ## Poison State Recovery /// /// This method automatically clears any poison state when successful. This is safe because: @@ -543,6 +569,15 @@ impl MultiUseSandbox { HyperlightVmError::Restore(e) })?; + // Restore captured MSR state. + #[cfg(target_arch = "x86_64")] + self.vm + .restore_msrs(snapshot.msrs(), snapshot.allowed_msrs()) + .map_err(|e| { + self.poisoned = true; + HyperlightVmError::Restore(e) + })?; + self.vm.set_stack_top(snapshot.stack_top_gva()); self.vm.set_entrypoint(snapshot.entrypoint()); @@ -2720,6 +2755,1223 @@ mod tests { ); } + #[cfg(target_arch = "x86_64")] + mod msr_tests { + use super::*; + use crate::HostFunctions; + use crate::hypervisor::hyperlight_vm::{CreateHyperlightVmError, HyperlightVmError}; + use crate::hypervisor::virtual_machine::{ + CreateVmError, RegisterError, ResetVcpuError, VmError, + }; + use crate::sandbox::snapshot::Snapshot; + + const KERNEL_GS_BASE: u32 = 0xC000_0102; + const SYSENTER_CS: u32 = 0x174; + + fn assert_msr_not_allowable(error: &HyperlightError, expected: u32) { + assert!( + matches!( + error, + HyperlightError::HyperlightVmError(HyperlightVmError::Create( + CreateHyperlightVmError::Vm(VmError::CreateVm( + CreateVmError::MsrNotAllowable { msr, .. } + )) + )) if *msr == expected + ), + "expected MsrNotAllowable for {expected:#x}, got: {error:?}" + ); + } + + fn assert_invalid_snapshot_msr(error: &HyperlightError) { + assert!( + matches!( + error, + HyperlightError::HyperlightVmError(HyperlightVmError::Restore( + ResetVcpuError::Register(RegisterError::InvalidSnapshotMsrIndex { .. }) + )) + ), + "expected InvalidSnapshotMsrIndex, got: {error:?}" + ); + } + + fn assert_msr_not_allowed(error: &HyperlightError) { + assert!( + matches!( + error, + HyperlightError::HyperlightVmError(HyperlightVmError::Restore( + ResetVcpuError::Register(RegisterError::SnapshotMsrNotAllowed { .. }) + )) + ), + "expected SnapshotMsrNotAllowed, got: {error:?}" + ); + } + + #[test] + fn kernel_gs_base_does_not_leak_through_swapgs() { + let mut sandbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let original: u64 = sandbox.call("ReadKernelGsBaseViaSwapgs", ()).unwrap(); + let sentinel = if original == 0x0000_7AAA_5555_AAAA { + 0x0000_6BBB_4444_BBBB + } else { + 0x0000_7AAA_5555_AAAA + }; + let snapshot = sandbox.snapshot().unwrap(); + + sandbox + .call::<()>("WriteKernelGsBaseViaSwapgs", sentinel) + .unwrap(); + assert_eq!( + sandbox + .call::("ReadKernelGsBaseViaSwapgs", ()) + .unwrap(), + sentinel + ); + + sandbox.restore(snapshot).unwrap(); + assert_eq!( + sandbox + .call::("ReadKernelGsBaseViaSwapgs", ()) + .unwrap(), + original, + "KERNEL_GS_BASE leaked across restore" + ); + } + + #[test] + fn snapshot_msr_values_survive_full_in_memory_lifecycle() { + let mut config = SandboxConfiguration::default(); + config.allow_msrs(&[KERNEL_GS_BASE]).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(config), + ) + .unwrap() + .evolve() + .unwrap(); + let first = 0x1111; + let second = 0x2222; + let third = 0x3333; + + source + .call::<()>("WriteMSR", (KERNEL_GS_BASE, first)) + .unwrap(); + assert_eq!( + source.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + first + ); + let first_snapshot = source.snapshot().unwrap(); + + source + .call::<()>("WriteMSR", (KERNEL_GS_BASE, second)) + .unwrap(); + assert_eq!( + source.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + second + ); + source.restore(first_snapshot.clone()).unwrap(); + assert_eq!( + source.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + first + ); + + let mut clone = MultiUseSandbox::from_snapshot( + first_snapshot.clone(), + HostFunctions::default(), + Some(config), + ) + .unwrap(); + assert_eq!(clone.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), first); + + clone + .call::<()>("WriteMSR", (KERNEL_GS_BASE, third)) + .unwrap(); + assert_eq!(clone.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), third); + let third_snapshot = clone.snapshot().unwrap(); + source.restore(third_snapshot.clone()).unwrap(); + assert_eq!( + source.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + third + ); + + let mut second_clone = MultiUseSandbox::from_snapshot( + third_snapshot, + HostFunctions::default(), + Some(config), + ) + .unwrap(); + assert_eq!( + second_clone.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + third + ); + second_clone.restore(first_snapshot).unwrap(); + assert_eq!( + second_clone.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + first + ); + } + + #[test] + fn equivalent_msr_configs_are_order_independent_across_sandboxes() { + let source_order = [KERNEL_GS_BASE, SYSENTER_CS]; + let target_order = [SYSENTER_CS, KERNEL_GS_BASE]; + let mut source_config = SandboxConfiguration::default(); + source_config.allow_msrs(&source_order).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(source_config), + ) + .unwrap() + .evolve() + .unwrap(); + source + .call::<()>("WriteMSR", (KERNEL_GS_BASE, 0x4444u64)) + .unwrap(); + assert_eq!( + source.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + 0x4444 + ); + source + .call::<()>("WriteMSR", (SYSENTER_CS, 0x5555u64)) + .unwrap(); + assert_eq!(source.call::("ReadMSR", SYSENTER_CS).unwrap(), 0x5555); + let snapshot = source.snapshot().unwrap(); + + let mut target_config = SandboxConfiguration::default(); + target_config.allow_msrs(&target_order).unwrap(); + let mut target = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(target_config), + ) + .unwrap() + .evolve() + .unwrap(); + target + .call::<()>("WriteMSR", (KERNEL_GS_BASE, 0xAAAAu64)) + .unwrap(); + assert_eq!( + target.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + 0xAAAA + ); + target + .call::<()>("WriteMSR", (SYSENTER_CS, 0xBBBBu64)) + .unwrap(); + assert_eq!(target.call::("ReadMSR", SYSENTER_CS).unwrap(), 0xBBBB); + target.restore(snapshot.clone()).unwrap(); + assert_eq!( + target.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + 0x4444 + ); + assert_eq!(target.call::("ReadMSR", SYSENTER_CS).unwrap(), 0x5555); + + let mut clone = MultiUseSandbox::from_snapshot( + snapshot, + HostFunctions::default(), + Some(target_config), + ) + .unwrap(); + assert_eq!( + clone.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + 0x4444 + ); + assert_eq!(clone.call::("ReadMSR", SYSENTER_CS).unwrap(), 0x5555); + } + + /// A restore succeeds when the destination allow list is a superset + /// of the snapshot's. The snapshot's allowed MSR keeps its captured + /// value. An MSR the destination adds resets to the baseline. + #[test] + fn snapshot_restores_into_superset_allow_list() { + const SYSENTER_ESP: u32 = 0x175; + let sentinel: u64 = 0x1234; + let mut source_config = SandboxConfiguration::default(); + source_config.allow_msrs(&[SYSENTER_CS]).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(source_config), + ) + .unwrap() + .evolve() + .unwrap(); + source + .call::<()>("WriteMSR", (SYSENTER_CS, sentinel)) + .unwrap(); + let snapshot = source.snapshot().unwrap(); + + let mut dest_config = SandboxConfiguration::default(); + dest_config + .allow_msrs(&[SYSENTER_CS, SYSENTER_ESP]) + .unwrap(); + + let mut clone = MultiUseSandbox::from_snapshot( + snapshot.clone(), + HostFunctions::default(), + Some(dest_config), + ) + .unwrap(); + assert_eq!(clone.call::("ReadMSR", SYSENTER_CS).unwrap(), sentinel); + let baseline: u64 = clone.call("ReadMSR", SYSENTER_ESP).unwrap(); + + let mut target = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(dest_config), + ) + .unwrap() + .evolve() + .unwrap(); + target + .call::<()>("WriteMSR", (SYSENTER_ESP, baseline ^ 0x55)) + .unwrap(); + target.restore(snapshot).unwrap(); + assert_eq!( + target.call::("ReadMSR", SYSENTER_CS).unwrap(), + sentinel + ); + // An MSR the destination adds resets to its baseline. + assert_eq!( + target.call::("ReadMSR", SYSENTER_ESP).unwrap(), + baseline + ); + } + + /// A restore is rejected when the snapshot allows an MSR the + /// destination does not. Both restore paths fail and poison the + /// sandbox. + #[test] + fn snapshot_rejects_non_superset_allow_list() { + const SYSENTER_ESP: u32 = 0x175; + let mut source_config = SandboxConfiguration::default(); + source_config.allow_msrs(&[SYSENTER_CS]).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(source_config), + ) + .unwrap() + .evolve() + .unwrap(); + source + .call::<()>("WriteMSR", (SYSENTER_CS, 0x1234u64)) + .unwrap(); + let snapshot = source.snapshot().unwrap(); + + // A destination that allows nothing, and one that allows a + // disjoint MSR, both fail the superset check. + for dest in [&[][..], &[SYSENTER_ESP][..]] { + let mut config = SandboxConfiguration::default(); + config.allow_msrs(dest).unwrap(); + + let err = MultiUseSandbox::from_snapshot( + snapshot.clone(), + HostFunctions::default(), + Some(config), + ) + .expect_err("from_snapshot must reject a non-superset allow list"); + assert_msr_not_allowed(&err); + + let mut target = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(config), + ) + .unwrap() + .evolve() + .unwrap(); + let err = target + .restore(snapshot.clone()) + .expect_err("restore must reject a non-superset allow list"); + assert_msr_not_allowed(&err); + assert!(target.poisoned()); + assert!(matches!( + target.call::("Echo", "hi".to_string()), + Err(HyperlightError::PoisonedSandbox) + )); + } + } + + #[test] + fn from_pre_init_snapshot_uses_local_msr_reset_set() { + let mut config = SandboxConfiguration::default(); + config.allow_msrs(&[KERNEL_GS_BASE]).unwrap(); + let snapshot = Arc::new( + Snapshot::from_env( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + config, + ) + .unwrap(), + ); + assert!(snapshot.msrs().is_none()); + + let mut sandbox = MultiUseSandbox::from_snapshot( + snapshot.clone(), + HostFunctions::default(), + Some(config), + ) + .unwrap(); + let baseline: u64 = sandbox.call("ReadMSR", KERNEL_GS_BASE).unwrap(); + sandbox + .call::<()>("WriteMSR", (KERNEL_GS_BASE, baseline ^ 0x55)) + .unwrap(); + assert_eq!( + sandbox.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + baseline ^ 0x55 + ); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn snapshot_without_msrs_uses_destination_reset_set() { + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + let snapshot = source.snapshot().unwrap(); + source.snapshot = None; + let Ok(mut snap) = Arc::try_unwrap(snapshot) else { + panic!("snapshot should be uniquely owned"); + }; + // A snapshot without MSRs uses the destination baseline. + snap.set_msrs(None); + snap.set_allowed_msrs(None); + let snapshot = Arc::new(snap); + + let mut config = SandboxConfiguration::default(); + config.allow_msrs(&[KERNEL_GS_BASE]).unwrap(); + let mut target = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(config), + ) + .unwrap() + .evolve() + .unwrap(); + let baseline: u64 = target.call("ReadMSR", KERNEL_GS_BASE).unwrap(); + target + .call::<()>("WriteMSR", (KERNEL_GS_BASE, baseline ^ 0x55)) + .unwrap(); + assert_eq!( + target.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + baseline ^ 0x55 + ); + target.restore(snapshot.clone()).unwrap(); + assert_eq!( + target.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + baseline + ); + target + .call::<()>("WriteMSR", (KERNEL_GS_BASE, baseline ^ 0xAA)) + .unwrap(); + assert_eq!( + target.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + baseline ^ 0xAA + ); + + let mut clone = + MultiUseSandbox::from_snapshot(snapshot, HostFunctions::default(), Some(config)) + .unwrap(); + let clone_baseline: u64 = clone.call("ReadMSR", KERNEL_GS_BASE).unwrap(); + clone + .call::<()>("WriteMSR", (KERNEL_GS_BASE, clone_baseline ^ 0xCC)) + .unwrap(); + assert_eq!( + clone.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + clone_baseline ^ 0xCC + ); + } + + #[test] + fn malformed_snapshot_msrs_poison_and_trusted_restore_recovers() { + let indices = [SYSENTER_CS, KERNEL_GS_BASE]; + let mut config = SandboxConfiguration::default(); + config.allow_msrs(&indices).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(config), + ) + .unwrap() + .evolve() + .unwrap(); + let snapshot = source.snapshot().unwrap(); + source.snapshot = None; + let Ok(mut snapshot) = Arc::try_unwrap(snapshot) else { + panic!("snapshot should be uniquely owned"); + }; + let mut msrs = snapshot.msrs().unwrap().clone(); + msrs[0].index = 0xDEAD; + snapshot.set_msrs(Some(msrs)); + let snapshot = Arc::new(snapshot); + + let error = MultiUseSandbox::from_snapshot( + snapshot.clone(), + HostFunctions::default(), + Some(config), + ) + .expect_err("from_snapshot must reject malformed snapshot MSRs"); + assert_invalid_snapshot_msr(&error); + + let mut target = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(config), + ) + .unwrap() + .evolve() + .unwrap(); + let trusted_value: u64 = target.call("ReadMSR", KERNEL_GS_BASE).unwrap(); + let recovery_snapshot = target.snapshot().unwrap(); + target + .call::<()>("WriteMSR", (KERNEL_GS_BASE, trusted_value ^ 0x55)) + .unwrap(); + let error = target + .restore(snapshot) + .expect_err("restore must reject malformed snapshot MSRs"); + assert_invalid_snapshot_msr(&error); + assert!(target.poisoned()); + assert!(matches!( + target.call::("Echo", "hi".to_string()), + Err(HyperlightError::PoisonedSandbox) + )); + + target.restore(recovery_snapshot).unwrap(); + assert!(!target.poisoned()); + assert_eq!( + target.call::("ReadMSR", KERNEL_GS_BASE).unwrap(), + trusted_value + ); + } + + #[test] + #[cfg(kvm)] + fn denied_msr_access_poisons_sandbox() { + use crate::hypervisor::virtual_machine::{HypervisorType, get_available_hypervisor}; + + match get_available_hypervisor() { + Some(HypervisorType::Kvm) => {} + _ => { + return; + } + } + + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let snapshot = sbox.snapshot().unwrap(); + let msr_index: u32 = 0xC000_0102; // IA32_KERNEL_GS_BASE + + let result = sbox.call::("ReadMSR", msr_index); + assert!( + matches!( + &result, + Err(HyperlightError::MsrReadViolation(idx)) if *idx == msr_index + ), + "RDMSR 0x{:X}: expected MsrReadViolation, got: {:?}", + msr_index, + result + ); + assert!(sbox.poisoned()); + + sbox.restore(snapshot.clone()).unwrap(); + + let result = sbox.call::<()>("WriteMSR", (msr_index, 0x5u64)); + assert!( + matches!( + &result, + Err(HyperlightError::MsrWriteViolation(idx, _)) if *idx == msr_index + ), + "WRMSR 0x{:X}: expected MsrWriteViolation, got: {:?}", + msr_index, + result + ); + assert!(sbox.poisoned()); + } + + /// A write-only command cannot enter the reset set. + #[test] + #[cfg(target_arch = "x86_64")] + fn test_allow_non_resettable_msr_fails_creation() { + let mut cfg = SandboxConfiguration::default(); + cfg.allow_msrs(&[0x49]).unwrap(); // IA32_PRED_CMD, a write-only command MSR + + let err = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(cfg), + ) + .unwrap() + .evolve() + .unwrap_err(); + + assert_msr_not_allowable(&err, 0x49); + } + + /// Host support cannot authorize an unclassified MSR. + #[test] + #[cfg(kvm)] + fn unclassified_allowed_msr_rejected_at_creation() { + use crate::hypervisor::virtual_machine::{HypervisorType, get_available_hypervisor}; + + if !matches!(get_available_hypervisor(), Some(HypervisorType::Kvm)) { + return; + } + + let mut cfg = SandboxConfiguration::default(); + cfg.allow_msrs(&[0x1A0]).unwrap(); // IA32_MISC_ENABLE: host-probeable, not in MSR_TABLE + + let err = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(cfg), + ) + .unwrap() + .evolve() + .expect_err("an unclassified allowed MSR must be rejected at creation"); + + assert_msr_not_allowable(&err, 0x1A0); + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn test_multiple_allowed_msrs_reset_across_restore() { + // Resettable MSRs the guest may write once allowed. + let msrs: [u32; 4] = [0x174, 0x175, 0x176, 0xC000_0102]; + let mut cfg = SandboxConfiguration::default(); + cfg.allow_msrs(&msrs).unwrap(); + + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(cfg), + ) + .unwrap() + .evolve() + .unwrap(); + + let baseline_snapshot = sbox.snapshot().unwrap(); + + let value: u64 = 0x1000; + for &msr in &msrs { + sbox.call::<()>("WriteMSR", (msr, value)).unwrap(); + let read_value: u64 = sbox.call("ReadMSR", msr).unwrap(); + assert_eq!(read_value, value, "MSR 0x{msr:X} should be writable"); + } + + sbox.restore(baseline_snapshot).unwrap(); + for &msr in &msrs { + let read_value: u64 = sbox.call("ReadMSR", msr).unwrap(); + assert_ne!( + read_value, value, + "MSR 0x{msr:X} should be reset to baseline across restore" + ); + } + } + + /// An allowed guest write must not survive restore. + #[test] + #[cfg(target_arch = "x86_64")] + fn test_allowed_msr_does_not_leak_across_restore() { + let msr_index: u32 = 0xC000_0102; // IA32_KERNEL_GS_BASE + let sentinel: u64 = 0xCAFE_F00D; + + let mut cfg = SandboxConfiguration::default(); + cfg.allow_msrs(&[msr_index]).unwrap(); + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + Some(cfg), + ) + .unwrap() + .evolve() + .unwrap(); + + let baseline = sbox.snapshot().unwrap(); + let original: u64 = sbox.call("ReadMSR", msr_index).unwrap(); + assert_ne!( + original, sentinel, + "test sentinel must differ from the baseline value" + ); + + sbox.call::<()>("WriteMSR", (msr_index, sentinel)).unwrap(); + assert_eq!( + sbox.call::("ReadMSR", msr_index).unwrap(), + sentinel, + "sentinel should be observable before restore" + ); + sbox.restore(baseline).unwrap(); + + let after: u64 = sbox.call("ReadMSR", msr_index).unwrap(); + assert_ne!(after, sentinel, "sentinel leaked across restore"); + assert_eq!(after, original, "MSR not reset to its baseline value"); + } + + /// KVM denies DEBUGCTL through its filter and x2APIC through xAPIC mode. + #[test] + #[cfg(all(kvm, target_arch = "x86_64"))] + fn test_debugctl_and_x2apic_msr_denied_by_default() { + use crate::hypervisor::virtual_machine::{HypervisorType, get_available_hypervisor}; + + if !matches!(get_available_hypervisor(), Some(HypervisorType::Kvm)) { + return; + } + + let cases: [(u32, bool); 2] = [(0x1D9, true), (0x800, false)]; + for (msr_index, expect_filter_violation) in cases { + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let result = sbox.call::<()>("WriteMSR", (msr_index, 0x1u64)); + if expect_filter_violation { + assert!( + matches!( + &result, + Err(HyperlightError::MsrWriteViolation(idx, _)) if *idx == msr_index + ), + "WRMSR 0x{msr_index:X}: expected MsrWriteViolation, got: {result:?}" + ); + } else { + assert!( + matches!(&result, Err(HyperlightError::GuestAborted(_, _))), + "WRMSR 0x{msr_index:X}: expected direct #GP, got: {result:?}" + ); + } + assert!( + sbox.poisoned(), + "sandbox should be poisoned after a denied WRMSR to 0x{msr_index:X}" + ); + } + } + + /// A rejected MSR restore poisons the sandbox on every backend. + /// + /// The host register set rejects the noncanonical KERNEL_GS_BASE value + /// on KVM, MSHV, and WHP alike, leaving the sandbox poisoned. + #[test] + #[cfg(target_arch = "x86_64")] + fn rejected_msr_restore_poisons_sandbox() { + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let baseline = sbox.snapshot().unwrap(); + sbox.snapshot = None; + let Ok(mut snap) = Arc::try_unwrap(baseline) else { + panic!("snapshot should be uniquely owned after clearing the cache"); + }; + // A noncanonical KERNEL_GS_BASE value the host set rejects. + let mut msrs = snap.msrs().unwrap().clone(); + let kernel_gs_base = msrs + .iter_mut() + .find(|entry| entry.index == 0xC000_0102) + .expect("KERNEL_GS_BASE should be in the reset set"); + kernel_gs_base.value = 0xDEAD_0000_0000_0000; + snap.set_msrs(Some(msrs)); + let snap = Arc::new(snap); + + let err = sbox + .restore(snap) + .expect_err("restore should fail on a rejected MSR set"); + assert!( + format!("{err:?}").to_lowercase().contains("msr") + || format!("{err:?}").to_lowercase().contains("restore"), + "expected an MSR restore error, got: {err:?}" + ); + assert!( + sbox.poisoned(), + "sandbox should be poisoned after failed restore" + ); + + let call = sbox.call::("Echo", "hi".to_string()); + assert!( + matches!(call, Err(HyperlightError::PoisonedSandbox)), + "poisoned sandbox should reject guest calls, got: {call:?}" + ); + } + + /// Unresettable feature-class MSRs must not retain guest writes. PMU, + /// LBR, and FRED are perfmon or feature gated. The AMD virtualization + /// MSRs are gated on nested-virt capability the sandbox never requests. + #[test] + #[cfg(target_arch = "x86_64")] + fn unresettable_msr_classes_do_not_leak() { + let cases: &[(u32, &str)] = &[ + (0xC1, "PMU IA32_PMC0"), + (0x186, "PMU IA32_PERFEVTSEL0"), + (0x38F, "PMU IA32_PERF_GLOBAL_CTRL"), + (0x1C8, "LBR_SELECT"), + (0x14CE, "arch-LBR IA32_LBR_CTL"), + (0x1D4, "FRED IA32_FRED_CONFIG"), + (0xC001_0114, "AMD VM_CR"), + (0xC001_0117, "AMD VM_HSAVE_PA"), + ]; + + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + for &(msr, _name) in cases { + assert_msr_write_does_not_survive_restore(&mut sbox, msr, 0x1); + } + } + + /// A guest write to IA32_MISC_ENABLE leaves no retained state. Hyper-V + /// drops the write on Intel and faults it on AMD. + #[test] + #[cfg(target_arch = "x86_64")] + fn misc_enable_guest_write_does_not_survive_restore() { + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + assert_msr_write_does_not_survive_restore(&mut sbox, 0x1A0, 1u64 << 40); + } + + /// Every stateful table entry needs runtime reset coverage. + #[test] + #[cfg(target_arch = "x86_64")] + fn runtime_msr_table_entries_are_justified() { + use crate::hypervisor::regs::core_reset_indices; + + #[cfg(kvm)] + let kernel_gs_uses_instruction_side_effect = matches!( + crate::hypervisor::virtual_machine::get_available_hypervisor(), + Some(crate::hypervisor::virtual_machine::HypervisorType::Kvm) + ); + #[cfg(not(kvm))] + let kernel_gs_uses_instruction_side_effect = false; + + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let reset_indices: Vec = sbox + .snapshot() + .unwrap() + .msrs() + .expect("filterless backend should have an MSR reset set") + .iter() + .map(|entry| entry.index) + .collect(); + + for index in core_reset_indices() { + if !reset_indices.contains(&index) { + assert_omitted_msr_does_not_retain(&mut sbox, index); + } else if kernel_gs_uses_instruction_side_effect && index == KERNEL_GS_BASE { + // Direct WRMSR is denied. The dedicated SWAPGS test proves + // the instruction-side mutation is restored. + } else if (index == 0x10 && !kernel_gs_uses_instruction_side_effect) + || matches!(index, 0xE7 | 0xE8) + { + assert_guest_counter_is_writable_and_restored(&mut sbox, index); + } else if let Some(sentinel) = positive_write_sentinel(index) { + assert_guest_msr_is_writable_and_restored(&mut sbox, index, sentinel); + } else { + assert!( + reset_exception_reason(index).is_some(), + "MSR 0x{index:X} is in the reset set without positive guest-write coverage or an explicit reason" + ); + } + } + } + + fn assert_omitted_msr_does_not_retain(sbox: &mut MultiUseSandbox, index: u32) { + let baseline = sbox.snapshot().unwrap(); + let original: u64 = match sbox.call("ReadMSR", index) { + Ok(value) => value, + Err(_) => { + assert!(sbox.poisoned(), "0x{index:X}: fault did not poison sandbox"); + sbox.restore(baseline).unwrap(); + return; + } + }; + let preferred = positive_write_sentinel(index).unwrap_or(original ^ 1); + let candidates = [preferred, original ^ 1, original ^ 2, 0, 1, 0x1000]; + + for candidate in candidates { + if candidate == original { + continue; + } + if sbox.call::<()>("WriteMSR", (index, candidate)).is_err() { + assert!(sbox.poisoned(), "0x{index:X}: fault did not poison sandbox"); + sbox.restore(baseline.clone()).unwrap(); + continue; + } + let written: u64 = sbox.call("ReadMSR", index).unwrap_or_else(|error| { + panic!("0x{index:X}: read after successful write failed: {error:?}") + }); + if written != original { + sbox.restore(baseline).unwrap(); + let after: u64 = sbox.call("ReadMSR", index).unwrap(); + assert_eq!( + after, original, + "0x{index:X}: guest retained a write but the MSR is absent from the reset set" + ); + return; + } + sbox.restore(baseline.clone()).unwrap(); + } + } + + fn positive_write_sentinel(index: u32) -> Option { + match index { + 0x174 => Some(0x10), // SYSENTER_CS + 0x175 | 0x176 => Some(0x1000), // SYSENTER_ESP/EIP + 0x277 => Some(0x0007_0406_0007_0406), // PAT + 0xC000_0081 => Some(0x001B_0008_0000_0000), // STAR + 0xC000_0082 | 0xC000_0083 => Some(0x1000), // LSTAR/CSTAR + 0xC000_0084 => Some(0x200), // SFMASK + 0xC000_0102 => Some(0x1000), // KERNEL_GS_BASE + 0x3B => Some(0x1000), // TSC_ADJUST + 0xC000_0103 => Some(0x5), // TSC_AUX + 0x2FF => Some(0xC00), // MTRR_DEF_TYPE + 0x200..=0x21F if index & 1 == 0 => Some(0x6), // MTRR_PHYSBASEn + 0x200..=0x21F => Some(0x800), // MTRR_PHYSMASKn + 0x250 | 0x258 | 0x259 | 0x268..=0x26F => Some(0x0606_0606_0606_0606), + _ => None, + } + } + + fn reset_exception_reason(index: u32) -> Option<&'static str> { + match index { + 0x10 => Some("KVM denies direct guest TSC MSR access"), + 0x1D9 => Some("DEBUGCTL support depends on exposed debug features"), + 0x48 => Some("SPEC_CTRL writable bits depend on mitigation features"), + 0x6A0 | 0x6A2 | 0x6A4..=0x6A8 => { + Some("CET writable state depends on exposed CET features") + } + 0x122 => Some("TSX_CTRL writable bits depend on exposed TSX features"), + 0x1C4 | 0x1C5 => Some("XFD writable bits depend on exposed XSAVE features"), + 0xE1 => Some("UMWAIT_CONTROL writable bits depend on exposed WAITPKG features"), + 0x6E0 => Some("TSC_DEADLINE writable bits depend on exposed APIC-timer features"), + 0xD90 => Some("BNDCFGS writable bits depend on exposed MPX features"), + 0xDA0 => Some("XSS writable bits depend on exposed XSAVE features"), + 0xC001_011F => { + Some("VIRT_SPEC_CTRL writable bits depend on exposed AMD SSBD virtualization") + } + _ => None, + } + } + + fn assert_guest_msr_is_writable_and_restored( + sbox: &mut MultiUseSandbox, + index: u32, + sentinel: u64, + ) { + let baseline = sbox.snapshot().unwrap(); + let original: u64 = sbox + .call("ReadMSR", index) + .unwrap_or_else(|error| panic!("0x{index:X}: guest RDMSR failed: {error:?}")); + let value = if original == sentinel { 0 } else { sentinel }; + + sbox.call::<()>("WriteMSR", (index, value)) + .unwrap_or_else(|error| panic!("0x{index:X}: guest WRMSR failed: {error:?}")); + let written: u64 = sbox + .call("ReadMSR", index) + .unwrap_or_else(|error| panic!("0x{index:X}: guest read-back failed: {error:?}")); + assert_eq!(written, value, "0x{index:X}: guest write did not stick"); + + sbox.restore(baseline).unwrap(); + let restored: u64 = sbox.call("ReadMSR", index).unwrap(); + assert_eq!( + restored, original, + "0x{index:X}: restore did not recover the baseline" + ); + } + + fn assert_guest_counter_is_writable_and_restored(sbox: &mut MultiUseSandbox, index: u32) { + let baseline = sbox.snapshot().unwrap(); + let original: u64 = sbox.call("ReadMSR", index).unwrap(); + let jump = original.wrapping_add(1 << 60); + + sbox.call::<()>("WriteMSR", (index, jump)).unwrap(); + let written: u64 = sbox.call("ReadMSR", index).unwrap(); + assert!( + written >= jump / 2, + "0x{index:X}: guest write did not stick" + ); + + sbox.restore(baseline).unwrap(); + let restored: u64 = sbox.call("ReadMSR", index).unwrap(); + assert!( + restored < jump / 2, + "0x{index:X}: restore did not pull the counter below the guest-written jump" + ); + } + + /// Verifies that a guest MSR write faults or resets to its baseline. + #[cfg(target_arch = "x86_64")] + fn assert_msr_write_does_not_survive_restore( + sbox: &mut MultiUseSandbox, + msr: u32, + sentinel: u64, + ) { + let baseline = sbox.snapshot().unwrap(); + let original: u64 = match sbox.call("ReadMSR", msr) { + Ok(v) => v, + Err(_) => { + assert!( + sbox.poisoned(), + "0x{msr:X}: a faulting RDMSR should poison the sandbox" + ); + sbox.restore(baseline).unwrap(); + return; + } + }; + assert_ne!( + original, sentinel, + "0x{msr:X}: sentinel must differ from baseline" + ); + + if sbox.call::<()>("WriteMSR", (msr, sentinel)).is_err() { + assert!( + sbox.poisoned(), + "0x{msr:X}: a faulting WRMSR should poison the sandbox" + ); + sbox.restore(baseline).unwrap(); + return; + } + + sbox.restore(baseline).unwrap(); + let after: u64 = sbox.call("ReadMSR", msr).unwrap(); + assert_eq!( + after, original, + "0x{msr:X}: MSR leaked across restore (expected 0x{original:X}, got 0x{after:X})" + ); + } + + /// Audits Hyper-V MSR bitmap ranges for guest state retained by restore. + #[test] + #[ignore = "slow host-dependent hardware MSR audit"] + #[cfg(target_arch = "x86_64")] + fn test_no_msr_leaks_across_restore_full_window_sweep() { + // Free-running counters use a magnitude check after restore. + const FREE_RUNNING: &[u32] = &[ + 0x10, // IA32_TIME_STAMP_COUNTER + 0xE7, // IA32_MPERF + 0xE8, // IA32_APERF + ]; + + #[cfg(kvm)] + if matches!( + crate::hypervisor::virtual_machine::get_available_hypervisor(), + Some(crate::hypervisor::virtual_machine::HypervisorType::Kvm) + ) { + return; + } + + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let baseline = sbox.snapshot().unwrap(); + + // At least one retained write must exercise restore. + let mut readable = 0usize; + let mut exercised: Vec = Vec::new(); + let mut read_only: Vec = Vec::new(); + let mut masked_only: Vec = Vec::new(); + // Collect all free-running leaks for one diagnostic. + let mut free_running_leaked: Vec = Vec::new(); + + // Architectural and low model-specific indices. + let low = 0x0000_0000u32..=0x0000_1FFF; + // Hyper-V synthetic indices. + let hyperv_synthetic = 0x4000_0000u32..=0x4000_1FFF; + // Extended and AMD model-specific indices. + let extended = 0xC000_0000u32..=0xC001_FFFF; + let windows = low.chain(hyperv_synthetic).chain(extended); + for msr in windows { + let original: u64 = match sbox.call("ReadMSR", msr) { + Ok(v) => v, + Err(_) => { + sbox.restore(baseline.clone()).unwrap(); + continue; + } + }; + readable += 1; + + // A large jump distinguishes reset from normal counter progress. + if FREE_RUNNING.contains(&msr) { + let jump = original.wrapping_add(1 << 60); + if sbox.call::<()>("WriteMSR", (msr, jump)).is_err() { + sbox.restore(baseline.clone()).unwrap(); + read_only.push(msr); + continue; + } + let planted = match sbox.call::("ReadMSR", msr) { + Ok(v) => v, + Err(_) => { + sbox.restore(baseline.clone()).unwrap(); + masked_only.push(msr); + continue; + } + }; + if planted < jump / 2 { + sbox.restore(baseline.clone()).unwrap(); + masked_only.push(msr); + continue; + } + sbox.restore(baseline.clone()).unwrap(); + let after: u64 = sbox.call("ReadMSR", msr).unwrap(); + if after < jump / 2 { + exercised.push(msr); + } else { + free_running_leaked.push(msr); + } + continue; + } + + // Multiple candidates cover MSRs with restricted writable bits. + let candidates = [ + original ^ 0x55, + original ^ 0x1, + original ^ (1 << 12), + original ^ (1 << 20), + original ^ (1 << 32), + original.wrapping_add(1), + 0, + ]; + let mut planted = false; + let mut saw_write = false; + for cand in candidates { + if cand == original { + continue; + } + if sbox.call::<()>("WriteMSR", (msr, cand)).is_err() { + sbox.restore(baseline.clone()).unwrap(); + continue; + } + saw_write = true; + match sbox.call::("ReadMSR", msr) { + Ok(v) if v != original => { + planted = true; + break; + } + _ => { + sbox.restore(baseline.clone()).unwrap(); + } + } + } + + if planted { + sbox.restore(baseline.clone()).unwrap(); + match sbox.call::("ReadMSR", msr) { + Ok(after) => assert_eq!( + after, original, + "0x{msr:X}: a guest MSR write leaked across restore \ + (expected 0x{original:X}, got 0x{after:X})" + ), + Err(e) => panic!("0x{msr:X}: read-back after restore failed: {e:?}"), + } + exercised.push(msr); + } else if saw_write { + masked_only.push(msr); + } else { + read_only.push(msr); + } + } + + let fmt = |v: &[u32]| { + v.iter() + .map(|m| format!("0x{m:X}")) + .collect::>() + .join(", ") + }; + eprintln!( + "full-window MSR sweep: readable={readable} exercised={} masked_only={} read_only={}", + exercised.len(), + masked_only.len(), + read_only.len() + ); + eprintln!(" exercised: [{}]", fmt(&exercised)); + eprintln!(" masked_only: [{}]", fmt(&masked_only)); + eprintln!(" read_only: [{}]", fmt(&read_only)); + eprintln!(" free_running_leaked: [{}]", fmt(&free_running_leaked)); + assert!( + free_running_leaked.is_empty(), + "free-running MSRs not reset across restore on this backend: [{}]", + fmt(&free_running_leaked) + ); + assert!( + !exercised.is_empty(), + "sweep was vacuous: no guest MSR write ever retained a value that restore \ + then rolled back, so the rollback path was never exercised" + ); + } + + /// A host TSC write must reset the guest-visible MSHV counter. + #[test] + #[cfg(all(mshv3, target_arch = "x86_64"))] + fn mshv_host_tsc_writeback_resets_guest_tsc() { + use crate::hypervisor::virtual_machine::{HypervisorType, get_available_hypervisor}; + + if !matches!(get_available_hypervisor(), Some(HypervisorType::Mshv)) { + return; + } + + let mut sbox = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")), + None, + ) + .unwrap() + .evolve() + .unwrap(); + + let base = sbox.vm.capture_msrs_for_test(&[0x10]).unwrap()[0].value; + + let jump = base.wrapping_add(1 << 60); + sbox.call::<()>("WriteMSR", (0x10u32, jump)).unwrap(); + let planted: u64 = sbox.call("ReadMSR", 0x10u32).unwrap(); + assert!( + planted >= jump, + "guest TSC write did not take (planted=0x{planted:X} jump=0x{jump:X})" + ); + + assert!( + sbox.vm.try_set_msr_for_test(0x10, base), + "host set of HV_X64_REGISTER_TSC failed" + ); + + let after: u64 = sbox.call("ReadMSR", 0x10u32).unwrap(); + eprintln!( + "mshv TSC writeback probe: base=0x{base:X} jump=0x{jump:X} planted=0x{planted:X} after=0x{after:X}" + ); + assert!( + after < jump / 2, + "host TSC write-back did NOT reset the guest TSC (after=0x{after:X} still near \ + jump=0x{jump:X}); the reset approach is not viable on this host" + ); + } + } + /// Tests for [`MultiUseSandbox::from_snapshot`] in-memory. mod from_snapshot { use std::sync::Arc; diff --git a/src/hyperlight_host/src/sandbox/snapshot/file/config.rs b/src/hyperlight_host/src/sandbox/snapshot/file/config.rs index 9b8d0792c..982489948 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file/config.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file/config.rs @@ -21,6 +21,8 @@ use serde::{Deserialize, Serialize}; use super::media_types::SNAPSHOT_ABI_VERSION; use crate::hypervisor::regs::CommonSpecialRegisters; +#[cfg(target_arch = "x86_64")] +use crate::hypervisor::regs::MsrEntry; use crate::mem::layout::SandboxMemoryLayout; // --- Arch and hypervisor identifiers -------------------------------- @@ -180,6 +182,17 @@ pub(super) struct OciSnapshotConfig { /// Special registers captured from the paused vCPU, restored /// verbatim when resuming the call. pub(super) sregs: CommonSpecialRegisters, + /// MSR reset state captured from the paused vCPU. `None` uses the + /// destination sandbox's reset set. + #[cfg(target_arch = "x86_64")] + #[serde(default)] + pub(super) msrs: Option>, + /// Allow list of the sandbox that captured this snapshot. Present + /// exactly when `msrs` is, and checked against the destination allow + /// list on load. + #[cfg(target_arch = "x86_64")] + #[serde(default)] + pub(super) allowed_msrs: Option>, pub(super) layout: MemoryLayout, /// Total size of the memory blob in bytes (including the guest /// page-table tail, if any). Equal to `self.memory.mem_size()`. @@ -592,6 +605,49 @@ mod tests { assert_eq!(restored, expected); } + /// Captured MSRs survive the serde round-trip through the config, + /// including index and value for every entry. + #[cfg(target_arch = "x86_64")] + #[test] + fn msrs_round_trip_preserves_every_entry() { + let original = gating_config_with_msrs(Some(vec![ + MsrEntry { + index: 0xC000_0102, + value: 0xDEAD_BEEF, + }, + MsrEntry { + index: 0x10, + value: 0x1234_5678_9ABC_DEF0, + }, + ])); + let json = serde_json::to_vec(&original).unwrap(); + let restored: OciSnapshotConfig = serde_json::from_slice(&json).unwrap(); + assert_eq!(restored.msrs, original.msrs); + } + + /// A config JSON with no MSR keys deserializes both fields to `None`. + #[cfg(target_arch = "x86_64")] + #[test] + fn config_without_msr_keys_deserializes_to_none() { + let with = gating_config_with_msrs(Some(vec![MsrEntry { + index: 0x10, + value: 1, + }])); + let mut json: serde_json::Value = + serde_json::from_slice(&serde_json::to_vec(&with).unwrap()).unwrap(); + assert!(json.as_object_mut().unwrap().remove("msrs").is_some()); + assert!( + json.as_object_mut() + .unwrap() + .remove("allowed_msrs") + .is_some() + ); + + let restored: OciSnapshotConfig = serde_json::from_value(json).unwrap(); + assert_eq!(restored.msrs, None); + assert_eq!(restored.allowed_msrs, None); + } + /// Every `ParameterType` survives the round-trip through its serde /// mirror, guarding against a transposed variant in either match. #[test] @@ -676,6 +732,10 @@ mod tests { stack_top_gva: 0x2000, entrypoint_addr: SandboxMemoryLayout::BASE_ADDRESS as u64, sregs: distinct_sregs(), + #[cfg(target_arch = "x86_64")] + msrs: None, + #[cfg(target_arch = "x86_64")] + allowed_msrs: None, layout: MemoryLayout { input_data_size: 0, output_data_size: 0, @@ -693,6 +753,17 @@ mod tests { } } + /// `gating_config` with a chosen MSR set, for serde tests. + #[cfg(target_arch = "x86_64")] + fn gating_config_with_msrs(msrs: Option>) -> OciSnapshotConfig { + let allowed_msrs = msrs.as_ref().map(|_| Vec::new()); + OciSnapshotConfig { + msrs, + allowed_msrs, + ..gating_config() + } + } + /// A snapshot built for a different architecture is rejected. #[test] fn validate_for_load_rejects_arch_mismatch() { @@ -888,6 +959,20 @@ mod schema_pin { 11 ] }, + "msrs": [ + { + "index": 16, + "value": 42 + }, + { + "index": 3221225474, + "value": 3735928559 + } + ], + "allowed_msrs": [ + 16, + 3221225474 + ], "layout": { "input_data_size": 1, "output_data_size": 2, diff --git a/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs b/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs index 59fcd6372..c97aadc2b 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs @@ -585,6 +585,10 @@ impl Snapshot { stack_top_gva: self.stack_top_gva, entrypoint_addr, sregs: *sregs, + #[cfg(target_arch = "x86_64")] + msrs: self.msrs.clone(), + #[cfg(target_arch = "x86_64")] + allowed_msrs: self.allowed_msrs.clone(), layout: MemoryLayout { input_data_size: l.input_data_size(), output_data_size: l.output_data_size(), @@ -850,6 +854,15 @@ impl Snapshot { // 8. Build entrypoint + sregs back from the config. let entrypoint = NextAction::Call(cfg.entrypoint_addr); + // `msrs` and `allowed_msrs` travel together. A config with one but + // not the other cannot enforce the allow-list check on restore. + #[cfg(target_arch = "x86_64")] + if cfg.msrs.is_some() != cfg.allowed_msrs.is_some() { + return Err(crate::new_error!( + "snapshot config inconsistent: msrs and allowed_msrs must both be present or both absent" + )); + } + // 9. Reconstitute host_functions metadata. let snapshot_generation = cfg.snapshot_generation; let host_funcs_vec: Vec< @@ -871,6 +884,10 @@ impl Snapshot { load_info: crate::mem::exe::LoadInfo::dummy(), stack_top_gva: cfg.stack_top_gva, sregs: Some(cfg.sregs), + #[cfg(target_arch = "x86_64")] + msrs: cfg.msrs, + #[cfg(target_arch = "x86_64")] + allowed_msrs: cfg.allowed_msrs, entrypoint, snapshot_generation, host_functions, diff --git a/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs b/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs index 691f6f920..bcce218f0 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs @@ -207,6 +207,200 @@ fn snapshot_generation_round_trip() { assert_eq!(loaded3.snapshot_generation(), gen3); } +/// A disk round-trip preserves captured MSR reset state. +#[cfg(target_arch = "x86_64")] +#[test] +fn msrs_round_trip_via_disk() { + let snap = create_snapshot(); + let original = snap.msrs().cloned(); + assert!( + original.as_ref().is_some_and(|m| !m.is_empty()), + "a running snapshot should capture a non-empty MSR reset set" + ); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("snap"); + snap.save(&path, &OciTag::new("latest").unwrap()).unwrap(); + + let loaded = Snapshot::checked_load(&path, OciTag::new("latest").unwrap()).unwrap(); + assert_eq!(loaded.msrs(), original.as_ref()); + // The allow list travels with the MSRs and is Some exactly when they are. + assert_eq!(loaded.allowed_msrs(), Some(&[][..])); +} + +/// A config with no `msrs` key loads and uses the destination reset set. +#[cfg(target_arch = "x86_64")] +#[test] +fn snapshot_without_msrs_key_loads_and_runs() { + let (_dir, path) = save_for_mutation(); + rewrite_config(&path, |cfg| { + let obj = cfg.as_object_mut().unwrap(); + assert!( + obj.remove("msrs").is_some(), + "a running x86_64 snapshot config should carry msrs to remove" + ); + assert!( + obj.remove("allowed_msrs").is_some(), + "a running x86_64 snapshot config should carry allowed_msrs to remove" + ); + }); + + let loaded = Snapshot::checked_load(&path, OciTag::new("latest").unwrap()).unwrap(); + assert_eq!(loaded.msrs(), None); + + let mut sbox = + MultiUseSandbox::from_snapshot(Arc::new(loaded), HostFunctions::default(), None).unwrap(); + assert_eq!(sbox.call::("GetStatic", ()).unwrap(), 0); +} + +/// A snapshot whose reset set includes an allowed MSR carries that +/// MSR's guest-written value across a disk round-trip. Loading with a +/// matching allow config restores the value, and restore-in-place +/// returns it after an overwrite. +#[cfg(target_arch = "x86_64")] +#[test] +fn disk_snapshot_restores_allowed_msr_value() { + const SYSENTER_CS: u32 = 0x174; + let sentinel: u64 = 0xDEAD_BEEF; + + let mut cfg = crate::sandbox::SandboxConfiguration::default(); + cfg.allow_msrs(&[SYSENTER_CS]).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().unwrap()), + Some(cfg), + ) + .unwrap() + .evolve() + .unwrap(); + source + .call::<()>("WriteMSR", (SYSENTER_CS, sentinel)) + .unwrap(); + let snap = source.snapshot().unwrap(); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("snap"); + snap.save(&path, &OciTag::new("latest").unwrap()).unwrap(); + let loaded = Arc::new(Snapshot::checked_load(&path, OciTag::new("latest").unwrap()).unwrap()); + + let mut cfg = crate::sandbox::SandboxConfiguration::default(); + cfg.allow_msrs(&[SYSENTER_CS]).unwrap(); + let mut sbox = + MultiUseSandbox::from_snapshot(loaded.clone(), HostFunctions::default(), Some(cfg)) + .unwrap(); + assert_eq!(sbox.call::("ReadMSR", SYSENTER_CS).unwrap(), sentinel); + + sbox.call::<()>("WriteMSR", (SYSENTER_CS, sentinel ^ 0x55)) + .unwrap(); + sbox.restore(loaded).unwrap(); + assert_eq!(sbox.call::("ReadMSR", SYSENTER_CS).unwrap(), sentinel); +} + +/// A disk snapshot restores into a destination with a superset allow list. +#[cfg(target_arch = "x86_64")] +#[test] +fn disk_snapshot_restores_into_superset_allow_list() { + const SYSENTER_CS: u32 = 0x174; + const SYSENTER_ESP: u32 = 0x175; + let sentinel: u64 = 0xDEAD_BEEF; + + let mut cfg = crate::sandbox::SandboxConfiguration::default(); + cfg.allow_msrs(&[SYSENTER_CS]).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().unwrap()), + Some(cfg), + ) + .unwrap() + .evolve() + .unwrap(); + source + .call::<()>("WriteMSR", (SYSENTER_CS, sentinel)) + .unwrap(); + let snap = source.snapshot().unwrap(); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("snap"); + snap.save(&path, &OciTag::new("latest").unwrap()).unwrap(); + let loaded = Arc::new(Snapshot::checked_load(&path, OciTag::new("latest").unwrap()).unwrap()); + + let mut dest_cfg = crate::sandbox::SandboxConfiguration::default(); + dest_cfg.allow_msrs(&[SYSENTER_CS, SYSENTER_ESP]).unwrap(); + let mut sbox = + MultiUseSandbox::from_snapshot(loaded, HostFunctions::default(), Some(dest_cfg)).unwrap(); + assert_eq!(sbox.call::("ReadMSR", SYSENTER_CS).unwrap(), sentinel); +} + +/// A disk snapshot is rejected when it allows an MSR the destination does not. +#[cfg(target_arch = "x86_64")] +#[test] +fn disk_snapshot_non_superset_allow_list_rejected() { + const SYSENTER_CS: u32 = 0x174; + let sentinel: u64 = 0x1234; + + let mut cfg = crate::sandbox::SandboxConfiguration::default(); + cfg.allow_msrs(&[SYSENTER_CS]).unwrap(); + let mut source = UninitializedSandbox::new( + GuestBinary::FilePath(simple_guest_as_string().unwrap()), + Some(cfg), + ) + .unwrap() + .evolve() + .unwrap(); + source + .call::<()>("WriteMSR", (SYSENTER_CS, sentinel)) + .unwrap(); + let snap = source.snapshot().unwrap(); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("snap"); + snap.save(&path, &OciTag::new("latest").unwrap()).unwrap(); + let loaded = Arc::new(Snapshot::checked_load(&path, OciTag::new("latest").unwrap()).unwrap()); + + // The destination allows nothing, so the snapshot's allowed MSR is not + // a subset of the destination allow list. + let err = MultiUseSandbox::from_snapshot(loaded.clone(), HostFunctions::default(), None) + .expect_err("loading into a non-superset allow list must fail"); + assert!( + format!("{err:?}").contains("SnapshotMsrNotAllowed"), + "expected an MSR allow-list mismatch, got: {err:?}" + ); + + let mut target = create_test_sandbox(); + let err = target + .restore(loaded) + .expect_err("restore into a non-superset allow list must fail"); + assert!( + format!("{err:?}").contains("SnapshotMsrNotAllowed"), + "expected an MSR allow-list mismatch, got: {err:?}" + ); + assert!(target.poisoned()); +} + +/// A config with `msrs` but no `allowed_msrs` cannot enforce the +/// allow-list check, so the load rejects it. +#[cfg(target_arch = "x86_64")] +#[test] +fn disk_snapshot_msrs_without_allow_list_rejected() { + let (_dir, path) = save_for_mutation(); + rewrite_config(&path, |cfg| { + assert!( + cfg.as_object_mut() + .unwrap() + .remove("allowed_msrs") + .is_some(), + "a running x86_64 snapshot config should carry allowed_msrs to remove" + ); + }); + + let err = unwrap_err_snapshot(Snapshot::checked_load( + &path, + OciTag::new("latest").unwrap(), + )); + assert!( + format!("{err:?}").contains("allowed_msrs"), + "expected an msrs/allowed_msrs consistency error, got: {err:?}" + ); +} + #[test] fn pre_init_snapshot_cannot_be_persisted() { let snap = Snapshot::from_env( diff --git a/src/hyperlight_host/src/sandbox/snapshot/mod.rs b/src/hyperlight_host/src/sandbox/snapshot/mod.rs index 9a638a98e..ac940cc59 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/mod.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/mod.rs @@ -32,6 +32,8 @@ use tracing::{Span, instrument}; use crate::Result; use crate::hypervisor::regs::CommonSpecialRegisters; +#[cfg(target_arch = "x86_64")] +use crate::hypervisor::regs::MsrEntry; use crate::mem::exe::{ExeInfo, LoadInfo}; use crate::mem::layout::SandboxMemoryLayout; use crate::mem::memory_region::{GuestMemoryRegion, MemoryRegion, MemoryRegionFlags}; @@ -92,6 +94,18 @@ pub struct Snapshot { /// tables are relocated during snapshot. sregs: Option, + /// MSR reset state captured from a running vCPU. + /// None for a pre-init snapshot or an old on-disk snapshot + /// predating MSR capture. + #[cfg(target_arch = "x86_64")] + msrs: Option>, + + /// The allow list of the sandbox that captured this snapshot, + /// sorted and deduplicated. Present exactly when `msrs` is present. + /// A restore requires the destination allow list to be a superset. + #[cfg(target_arch = "x86_64")] + allowed_msrs: Option>, + /// The next action that should be performed on this snapshot entrypoint: NextAction, @@ -381,6 +395,10 @@ impl Snapshot { load_info, stack_top_gva: exn_stack_top_gva, sregs: None, + #[cfg(target_arch = "x86_64")] + msrs: None, + #[cfg(target_arch = "x86_64")] + allowed_msrs: None, entrypoint: NextAction::Initialise(load_addr + entrypoint_va - base_va), snapshot_generation: 0, host_functions: HostFunctionDetails { @@ -407,6 +425,8 @@ impl Snapshot { root_pt_gpas: &[u64], stack_top_gva: u64, sregs: CommonSpecialRegisters, + #[cfg(target_arch = "x86_64")] msrs: Option>, + #[cfg(target_arch = "x86_64")] allowed_msrs: Option>, entrypoint: NextAction, snapshot_generation: u64, host_functions: HostFunctionDetails, @@ -560,6 +580,10 @@ impl Snapshot { load_info, stack_top_gva, sregs: Some(sregs), + #[cfg(target_arch = "x86_64")] + msrs, + #[cfg(target_arch = "x86_64")] + allowed_msrs, entrypoint, snapshot_generation, host_functions, @@ -603,6 +627,30 @@ impl Snapshot { self.sregs.as_ref() } + /// Returns the captured MSR reset state. + #[cfg(target_arch = "x86_64")] + pub(crate) fn msrs(&self) -> Option<&Vec> { + self.msrs.as_ref() + } + + /// Returns the capturing sandbox's allow list. + #[cfg(target_arch = "x86_64")] + pub(crate) fn allowed_msrs(&self) -> Option<&[u32]> { + self.allowed_msrs.as_deref() + } + + /// Stores captured MSR reset state. + #[cfg(all(test, target_arch = "x86_64"))] + pub(crate) fn set_msrs(&mut self, msrs: Option>) { + self.msrs = msrs; + } + + /// Stores the capturing sandbox's allow list. + #[cfg(all(test, target_arch = "x86_64"))] + pub(crate) fn set_allowed_msrs(&mut self, allowed: Option>) { + self.allowed_msrs = allowed; + } + pub(crate) fn entrypoint(&self) -> NextAction { self.entrypoint } @@ -761,6 +809,10 @@ mod tests { &[pt_base], 0, default_sregs(), + #[cfg(target_arch = "x86_64")] + None, + #[cfg(target_arch = "x86_64")] + None, super::NextAction::None, 1, HostFunctionDetails::default(), @@ -778,6 +830,10 @@ mod tests { &[pt_base], 0, default_sregs(), + #[cfg(target_arch = "x86_64")] + None, + #[cfg(target_arch = "x86_64")] + None, super::NextAction::None, 2, HostFunctionDetails::default(), diff --git a/src/tests/rust_guests/simpleguest/src/main.rs b/src/tests/rust_guests/simpleguest/src/main.rs index 2da2eb7ca..efe6badab 100644 --- a/src/tests/rust_guests/simpleguest/src/main.rs +++ b/src/tests/rust_guests/simpleguest/src/main.rs @@ -1095,6 +1095,87 @@ fn call_host_expect_error(hostfuncname: String) -> Result<()> { Ok(()) } +#[guest_function("ReadMSR")] +#[cfg(target_arch = "x86_64")] +fn read_msr(msr: u32) -> u64 { + let (read_eax, read_edx): (u32, u32); + // SAFETY: The test guest runs at CPL0, and the operands declare every register used. + unsafe { + core::arch::asm!( + "rdmsr", + in("ecx") msr, + out("eax") read_eax, + out("edx") read_edx, + options(nostack, nomem) + ); + } + ((read_edx as u64) << 32) | (read_eax as u64) +} + +#[guest_function("WriteKernelGsBaseViaSwapgs")] +#[cfg(target_arch = "x86_64")] +fn write_kernel_gs_base_via_swapgs(value: u64) { + // SAFETY: The test guest runs at CPL0, and CR4 is restored before returning. + unsafe { + core::arch::asm!( + "mov {original_cr4}, cr4", + "mov {enabled_cr4}, {original_cr4}", + "or {enabled_cr4}, {fsgsbase}", + "mov cr4, {enabled_cr4}", + "wrgsbase {value}", + "swapgs", + "mov cr4, {original_cr4}", + original_cr4 = out(reg) _, + enabled_cr4 = out(reg) _, + fsgsbase = const 1 << 16, + value = in(reg) value, + options(nostack) + ); + } +} + +#[guest_function("ReadKernelGsBaseViaSwapgs")] +#[cfg(target_arch = "x86_64")] +fn read_kernel_gs_base_via_swapgs() -> u64 { + let value: u64; + // SAFETY: The test guest runs at CPL0, and SWAPGS and CR4 are restored before returning. + unsafe { + core::arch::asm!( + "mov {original_cr4}, cr4", + "mov {enabled_cr4}, {original_cr4}", + "or {enabled_cr4}, {fsgsbase}", + "mov cr4, {enabled_cr4}", + "swapgs", + "rdgsbase {value}", + "swapgs", + "mov cr4, {original_cr4}", + original_cr4 = out(reg) _, + enabled_cr4 = out(reg) _, + value = lateout(reg) value, + fsgsbase = const 1 << 16, + options(nostack) + ); + } + value +} + +#[guest_function("WriteMSR")] +#[cfg(target_arch = "x86_64")] +fn write_msr(msr: u32, value: u64) { + let eax = (value & 0xFFFFFFFF) as u32; + let edx = ((value >> 32) & 0xFFFFFFFF) as u32; + // SAFETY: The test guest runs at CPL0, and the operands declare every register used. + unsafe { + core::arch::asm!( + "wrmsr", + in("ecx") msr, + in("eax") eax, + in("edx") edx, + options(nostack, nomem) + ); + } +} + #[hyperlight_guest_bin::main] #[instrument(skip_all, parent = Span::current(), level= "Trace")] fn main() {