From 8a9c819297fb583d40c62bcc2af57660e3c663dd Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 5 Aug 2026 22:30:54 +0800 Subject: [PATCH 1/2] vm: exempt clone/restore provisioning from the child VM's CPU ceiling (#186) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since #182 the VMM pays its own Guaranteed-at-N quota from the first instruction, so restore's eager memory copy runs on 1 core of budget: under a 20-VM storm, copy-mode clone P90 doubled (530 -> 1032ms). Clone/restore now launch with cpu.max at max — weight, fence, and placement still bind, so loading competes fairly inside the fence rather than running unbounded — and the finite quota (then burst) is armed after the memory load, before resume. Every pre-arm failure leaves a paused VMM with no guest work done; after resume the cap is already live, so no uncapped-running state exists to converge. Plain boot keeps arming at spawn. --- cgroup/cgroup.go | 30 ++++++++++++++++++------ cgroup/cgroup_test.go | 21 +++++++++++++++++ docs/vm.md | 2 ++ hypervisor/backend.go | 3 +++ hypervisor/cloudhypervisor/clone.go | 7 +++++- hypervisor/cloudhypervisor/clone_test.go | 18 +++++++++++++- hypervisor/cloudhypervisor/restore.go | 5 +++- hypervisor/cloudhypervisor/start.go | 11 +++++---- hypervisor/firecracker/clone.go | 9 +++++-- hypervisor/firecracker/restore.go | 5 +++- hypervisor/firecracker/start.go | 17 +++++++------- hypervisor/start.go | 10 +++++++- 12 files changed, 111 insertions(+), 27 deletions(-) diff --git a/cgroup/cgroup.go b/cgroup/cgroup.go index 5416822b..a456b6e5 100644 --- a/cgroup/cgroup.go +++ b/cgroup/cgroup.go @@ -133,8 +133,8 @@ func ScopeDir(parentDir, vmID string) string { return filepath.Join(parentDir, scopePrefix+vmID+scopeSuffix) } -// Prepare creates or reconfigures vmID's scope and returns its opened directory for CLONE_INTO_CGROUP; idempotent, so a relaunch reuses a scope its dying predecessor still occupies. -func Prepare(parentDir, fence, vmID string, k Knobs) (*os.File, error) { +// Prepare creates or reconfigures vmID's scope and returns its opened directory for CLONE_INTO_CGROUP; idempotent, so a relaunch reuses a scope its dying predecessor still occupies. deferQuota leaves the ceiling at max for the provisioning window — weight, fence, and placement still apply — until Arm sets the finite quota. +func Prepare(parentDir, fence, vmID string, k Knobs, deferQuota bool) (*os.File, error) { if vmID == "" { return nil, errors.New("cgroup scope: empty vm id") } @@ -158,13 +158,12 @@ func Prepare(parentDir, fence, vmID string, k Knobs) (*os.File, error) { return nil, err } } - if err := writeControl(dir, maxName, fmt.Sprintf("%d %d", k.QuotaUs, k.PeriodUs)); err != nil { - return nil, err - } - if k.BurstUs > 0 { - if err := writeControl(dir, burstName, strconv.FormatInt(k.BurstUs, 10)); err != nil { + if deferQuota { + if err := writeControl(dir, maxName, fmt.Sprintf("max %d", k.PeriodUs)); err != nil { return nil, err } + } else if err := armQuota(dir, k); err != nil { + return nil, err } scope, err := os.Open(dir) //nolint:gosec // path derives from config parent + generated VM ID if err != nil { @@ -173,6 +172,23 @@ func Prepare(parentDir, fence, vmID string, k Knobs) (*os.File, error) { return scope, nil } +// Arm sets the finite quota on a scope prepared with deferQuota; called after memory load, before resume, so the guest never runs uncapped. Idempotent — a retry after a partial failure converges. +func Arm(parentDir, vmID string, k Knobs) error { + return armQuota(ScopeDir(parentDir, vmID), k) +} + +func armQuota(dir string, k Knobs) error { + if err := writeControl(dir, maxName, fmt.Sprintf("%d %d", k.QuotaUs, k.PeriodUs)); err != nil { + return err + } + if k.BurstUs > 0 { + if err := writeControl(dir, burstName, strconv.FormatInt(k.BurstUs, 10)); err != nil { + return err + } + } + return nil +} + // Remove kills everything left in an owned scope and removes it; the VMM must already be confirmed dead. ENOENT counts as success. func Remove(ctx context.Context, parentDir, vmID string) error { dir := ScopeDir(parentDir, vmID) diff --git a/cgroup/cgroup_test.go b/cgroup/cgroup_test.go index e210f459..1eaa597c 100644 --- a/cgroup/cgroup_test.go +++ b/cgroup/cgroup_test.go @@ -235,3 +235,24 @@ func TestEffectiveCPUs(t *testing.T) { t.Error("no constraint: want nil") } } + +func TestArmSetsQuotaThenBurst(t *testing.T) { + parent := t.TempDir() + dir := ScopeDir(parent, "A") + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatalf("setup: %v", err) + } + for _, f := range []string{"cpu.max", "cpu.max.burst"} { + if err := os.WriteFile(filepath.Join(dir, f), nil, 0o600); err != nil { + t.Fatalf("setup: %v", err) + } + } + if err := Arm(parent, "A", Knobs{QuotaUs: 150000, PeriodUs: 100000, BurstUs: 50000}); err != nil { + t.Fatalf("Arm: %v", err) + } + max, _ := os.ReadFile(filepath.Join(dir, "cpu.max")) + burst, _ := os.ReadFile(filepath.Join(dir, "cpu.max.burst")) + if string(max) != "150000 100000" || string(burst) != "50000" { + t.Errorf("max=%q burst=%q", max, burst) + } +} diff --git a/docs/vm.md b/docs/vm.md index 0af6c229..929a0c85 100644 --- a/docs/vm.md +++ b/docs/vm.md @@ -37,6 +37,8 @@ Defaults are Kubernetes-style Guaranteed at N for `--cpu N`: quota = N cores (`- `cgroup_cpus` fences the whole VM population onto a host cpu subset (e.g. `0-14` on a 16-core host keeps core 15 for the OS, the API consumer, and clone/wake execution). `--cpuset-cpus` pins one VM to specific cores inside the fence. Both are validated by cocoon against the effective sets — the kernel silently degrades ungrantable cpuset requests rather than failing — and shrinking the fence is refused while a running VM's placement conflicts. Clearing `cgroup_cpus` converges: the stale fence is reset on the next launch. +Provisioning is exempt from the ceiling: clone/restore launch with the quota at `max` — weight, fence, and placement still apply — and the finite quota is armed after the memory load completes, before the guest resumes, so snapshot loading runs at control-plane speed while the guest never executes uncapped. The mmap restore mode's deferred first-touch faults are inherent guest-lifetime work and do pay the VM's quota — on dense hosts prefer `copy` mode for latency-critical wakes or grant quota headroom. + cgroup knobs are host-side policy, like networking: snapshots record the source VM's values but never apply them — a clone takes its policy from flags (defaults otherwise), restore keeps the target VM's. Scopes are removed when the VMM dies (stop, hibernate, delete, crash convergence) and orphans are swept by `cocoon gc`. `cocoon vm list` shows per-VM throttling as `THROTTLED` (`nr_throttled/throttled_usec` from `cpu.stat`). Requirements: cgroup v2 unified hierarchy with the `cpu` controller (kernel ≥ 5.14 for burst), running cocoon as root (production shape). Non-root works inside a systemd user slice with delegated controllers (`systemd-run --user --scope`), where user slices typically delegate `cpu` but not `cpuset` — fence/placement then fail preflight with the exact missing file named. diff --git a/hypervisor/backend.go b/hypervisor/backend.go index d033eb70..879ad707 100644 --- a/hypervisor/backend.go +++ b/hypervisor/backend.go @@ -106,6 +106,9 @@ type LaunchSpec struct { // Rec names the VM whose CPU scope the process enters at spawn (ID + cgroup knobs); every VM enters a scope. Rec *VMRecord + + // DeferCPUQuota leaves the scope's ceiling at max through the paused provisioning window (#186); the backend arms the finite quota before resume. + DeferCPUQuota bool } // PreflightHook validates rec against the snapshot source dir before anything is applied. diff --git a/hypervisor/cloudhypervisor/clone.go b/hypervisor/cloudhypervisor/clone.go index f7327ee1..7603fafc 100644 --- a/hypervisor/cloudhypervisor/clone.go +++ b/hypervisor/cloudhypervisor/clone.go @@ -22,6 +22,7 @@ import ( const restoreTAPPrefix = "rm" type cloneResumeOpts struct { + vmID string vmCfg *types.VMConfig directBoot bool hadCidataInSnapshot bool @@ -130,13 +131,14 @@ func (ch *CloudHypervisor) cloneAfterExtractParsed(ctx context.Context, vmID str VM: types.VM{ID: vmID, Config: *vmCfg}, RunDir: runDir, LogDir: logDir, - }, args, net.NetnsPath) + }, args, net.NetnsPath, true) if err != nil { ch.MarkError(ctx, vmID) return nil, fmt.Errorf("launch CH: %w", err) } if err := ch.restoreAndResumeClone(ctx, pid, sockPath, runDir, &cloneResumeOpts{ + vmID: vmID, vmCfg: vmCfg, directBoot: directBoot, hadCidataInSnapshot: hadCidataInSnapshot, @@ -198,6 +200,9 @@ func (ch *CloudHypervisor) restoreAndResumeClone(ctx context.Context, pid int, s return fmt.Errorf("vm.add-disk (data %s): %w", sc.Serial, err) } } + if err = ch.ArmCPUQuota(opts.vmID, &opts.vmCfg.Config); err != nil { + return err + } if err = resumeVM(ctx, hc); err != nil { return fmt.Errorf("vm.resume: %w", err) } diff --git a/hypervisor/cloudhypervisor/clone_test.go b/hypervisor/cloudhypervisor/clone_test.go index 632b3df3..967cccbd 100644 --- a/hypervisor/cloudhypervisor/clone_test.go +++ b/hypervisor/cloudhypervisor/clone_test.go @@ -363,8 +363,16 @@ func TestRestoreAndResumeCloneHotplugsCidataByRole(t *testing.T) { {Path: "/run/cidata.img", RO: true, Role: types.StorageRoleCidata}, {Path: "/run/extra.raw", Role: types.StorageRoleData, Serial: "extra"}, } - ch := &CloudHypervisor{conf: NewConfig(&config.Config{})} + scopeParent := t.TempDir() + if err := os.Mkdir(filepath.Join(scopeParent, "vm-T.scope"), 0o755); err != nil { + t.Fatalf("setup: %v", err) + } + ch := &CloudHypervisor{ + Backend: &hypervisor.Backend{Conf: tmpScopeConf{Config: NewConfig(&config.Config{}), parent: scopeParent}}, + conf: NewConfig(&config.Config{}), + } if err := ch.restoreAndResumeClone(t.Context(), 0, sock, t.TempDir(), &cloneResumeOpts{ + vmID: "T", vmCfg: &types.VMConfig{Config: types.Config{CPU: 2}}, storageConfigs: storageConfigs, dataDisks: storageConfigs[2:], @@ -495,3 +503,11 @@ func basePatchOpts() *patchOptions { directBoot: true, } } + +// tmpScopeConf redirects the cgroup parent to a temp dir so Arm writes plain files. +type tmpScopeConf struct { + *Config + parent string +} + +func (c tmpScopeConf) CgroupParentDir() string { return c.parent } diff --git a/hypervisor/cloudhypervisor/restore.go b/hypervisor/cloudhypervisor/restore.go index b2ab8db9..056b17a1 100644 --- a/hypervisor/cloudhypervisor/restore.go +++ b/hypervisor/cloudhypervisor/restore.go @@ -96,7 +96,7 @@ func (ch *CloudHypervisor) restoreAfterExtract(ctx context.Context, vmID string, // Launch under the target config: a --force cross-config restore changes the vCPU count the scope derives from. rec.Config = *vmCfg - pid, launchErr := ch.launchProcess(ctx, rec, args, rec.ResolvedNetnsPath()) + pid, launchErr := ch.launchProcess(ctx, rec, args, rec.ResolvedNetnsPath(), true) if launchErr != nil { return nil, fmt.Errorf("launch CH: %w", launchErr) } @@ -112,6 +112,9 @@ func (ch *CloudHypervisor) restoreAfterExtract(ctx context.Context, vmID string, if err = restoreVM(ctx, hc, rec.RunDir, vmCfg.RestoreMode); err != nil { return nil, fmt.Errorf("vm.restore: %w", err) } + if err = ch.ArmCPUQuota(vmID, &vmCfg.Config); err != nil { + return nil, err + } if err = resumeVM(ctx, hc); err != nil { return nil, fmt.Errorf("vm.resume: %w", err) } diff --git a/hypervisor/cloudhypervisor/start.go b/hypervisor/cloudhypervisor/start.go index 59c81d9e..61e7e980 100644 --- a/hypervisor/cloudhypervisor/start.go +++ b/hypervisor/cloudhypervisor/start.go @@ -23,12 +23,12 @@ func (ch *CloudHypervisor) startOne(ctx context.Context, id string) error { vmCfg := buildVMConfig(rec, hypervisor.ConsoleSockPath(rec.RunDir), cgroup.EffectiveCPUs(rec.Config.CPUSetCPUs, ch.conf.CgroupCPUs)) args := buildCLIArgs(vmCfg, sockPath) ch.saveCmdline(ctx, rec, args) - return ch.launchProcess(ctx, rec, args, rec.ResolvedNetnsPath()) + return ch.launchProcess(ctx, rec, args, rec.ResolvedNetnsPath(), false) }, }) } -func (ch *CloudHypervisor) launchProcess(ctx context.Context, rec *hypervisor.VMRecord, args []string, netnsPath string) (int, error) { +func (ch *CloudHypervisor) launchProcess(ctx context.Context, rec *hypervisor.VMRecord, args []string, netnsPath string, deferQuota bool) (int, error) { processLog := ch.LogFilePath(rec.LogDir) logFile, err := os.Create(processLog) //nolint:gosec if err != nil { @@ -47,9 +47,10 @@ func (ch *CloudHypervisor) launchProcess(ctx context.Context, rec *hypervisor.VM } pid, err := ch.LaunchVMProcess(ctx, hypervisor.LaunchSpec{ - Cmd: cmd, - NetnsPath: netnsPath, - Rec: rec, + Cmd: cmd, + NetnsPath: netnsPath, + Rec: rec, + DeferCPUQuota: deferQuota, }) if err != nil { return 0, err diff --git a/hypervisor/firecracker/clone.go b/hypervisor/firecracker/clone.go index cc1d7a91..259f1581 100644 --- a/hypervisor/firecracker/clone.go +++ b/hypervisor/firecracker/clone.go @@ -49,6 +49,8 @@ type launchCloneFn func([]*os.File) (int, *cloneLeaseControl, error) type cloneLaunch struct { launch launchCloneFn sockPath, runDir string + vmID string + vmCfg *types.VMConfig networkConfigs []*types.NetworkConfig src, dst []*types.StorageConfig } @@ -101,10 +103,10 @@ func (fc *Firecracker) cloneAfterExtract(ctx context.Context, vmID string, vmCfg VM: types.VM{ID: vmID, Config: *vmCfg}, RunDir: runDir, LogDir: logDir, - }, sockPath, net.NetnsPath, leaseFiles) + }, sockPath, net.NetnsPath, leaseFiles, true) } pid, leaseControl, plan, cloneErr := fc.startCloneVM(ctx, cloneLaunch{ - launch: launch, sockPath: sockPath, runDir: runDir, + launch: launch, sockPath: sockPath, runDir: runDir, vmID: vmID, vmCfg: vmCfg, networkConfigs: networkConfigs, src: meta.StorageConfigs, dst: storageConfigs, }) if cloneErr != nil { @@ -200,6 +202,9 @@ func (fc *Firecracker) resumeAndReanchorClone(ctx context.Context, pid int, cl c } }() + if err = fc.ArmCPUQuota(cl.vmID, &cl.vmCfg.Config); err != nil { + return err + } hc := utils.NewSocketHTTPClient(cl.sockPath) if err = resumeVM(ctx, hc); err != nil { return fmt.Errorf("resume: %w", err) diff --git a/hypervisor/firecracker/restore.go b/hypervisor/firecracker/restore.go index 1444f27a..6b978886 100644 --- a/hypervisor/firecracker/restore.go +++ b/hypervisor/firecracker/restore.go @@ -63,7 +63,7 @@ func (fc *Firecracker) restoreAfterExtract(ctx context.Context, vmID string, vmC // Launch under the target config: a --force cross-config restore changes the vCPU count the scope derives from. rec.Config = *vmCfg - pid, launchErr := fc.launchProcess(ctx, rec, sockPath, rec.ResolvedNetnsPath()) + pid, launchErr := fc.launchProcess(ctx, rec, sockPath, rec.ResolvedNetnsPath(), true) if launchErr != nil { return nil, fmt.Errorf("launch FC: %w", launchErr) } @@ -79,6 +79,9 @@ func (fc *Firecracker) restoreAfterExtract(ctx context.Context, vmID string, vmC return nil, fmt.Errorf("snapshot/load: %w", err) } + if err = fc.ArmCPUQuota(vmID, &vmCfg.Config); err != nil { + return nil, err + } hc := utils.NewSocketHTTPClient(sockPath) if err = resumeVM(ctx, hc); err != nil { return nil, fmt.Errorf("resume: %w", err) diff --git a/hypervisor/firecracker/start.go b/hypervisor/firecracker/start.go index 770ef8d9..4b12d219 100644 --- a/hypervisor/firecracker/start.go +++ b/hypervisor/firecracker/start.go @@ -32,7 +32,7 @@ func (fc *Firecracker) startOne(ctx context.Context, id string) error { return fc.StartSequence(ctx, id, hypervisor.StartSpec{ RuntimeFiles: runtimeFiles, Launch: func(ctx context.Context, rec *hypervisor.VMRecord, sockPath string) (int, error) { - return fc.launchProcess(ctx, rec, sockPath, rec.ResolvedNetnsPath()) + return fc.launchProcess(ctx, rec, sockPath, rec.ResolvedNetnsPath(), false) }, PostLaunch: func(ctx context.Context, rec *hypervisor.VMRecord, sockPath string, _ int) error { return fc.configureVM(ctx, utils.NewSocketHTTPClient(sockPath), rec) @@ -149,13 +149,13 @@ func (c *cloneLeaseControl) commit() error { return waitRelayLeaseResponse(c.responses, relayLeaseCommitAck, "commit") } -func (fc *Firecracker) launchProcess(ctx context.Context, rec *hypervisor.VMRecord, sockPath, netnsPath string) (int, error) { - pid, _, err := fc.launchProcessWithLeases(ctx, rec, sockPath, netnsPath, nil) +func (fc *Firecracker) launchProcess(ctx context.Context, rec *hypervisor.VMRecord, sockPath, netnsPath string, deferQuota bool) (int, error) { + pid, _, err := fc.launchProcessWithLeases(ctx, rec, sockPath, netnsPath, nil, deferQuota) return pid, err } // launchProcessWithLeases keeps inherited source-VM leases in the console relay until the caller confirms clone setup. -func (fc *Firecracker) launchProcessWithLeases(ctx context.Context, rec *hypervisor.VMRecord, sockPath, netnsPath string, leaseFiles []*os.File) (int, *cloneLeaseControl, error) { +func (fc *Firecracker) launchProcessWithLeases(ctx context.Context, rec *hypervisor.VMRecord, sockPath, netnsPath string, leaseFiles []*os.File, deferQuota bool) (int, *cloneLeaseControl, error) { logger := log.WithFunc("firecracker.launchProcessWithLeases") fcLog := fc.LogFilePath(rec.LogDir) @@ -183,10 +183,11 @@ func (fc *Firecracker) launchProcessWithLeases(ctx context.Context, rec *hypervi fcCmd.Stdin = slave fcCmd.Stdout = slave pid, err := fc.LaunchVMProcess(ctx, hypervisor.LaunchSpec{ - Cmd: fcCmd, - NetnsPath: netnsPath, - OnFail: func() { _ = master.Close() }, - Rec: rec, + Cmd: fcCmd, + NetnsPath: netnsPath, + OnFail: func() { _ = master.Close() }, + Rec: rec, + DeferCPUQuota: deferQuota, }) if err != nil { return 0, nil, err diff --git a/hypervisor/start.go b/hypervisor/start.go index 01113d1d..dfe2f6d1 100644 --- a/hypervisor/start.go +++ b/hypervisor/start.go @@ -134,7 +134,7 @@ func (b *Backend) LaunchVMProcess(ctx context.Context, spec LaunchSpec) (pid int } }() - scope, err := cgroup.Prepare(b.Conf.CgroupParentDir(), b.Conf.CgroupCPUFence(), spec.Rec.ID, cgroup.ResolveKnobs(&spec.Rec.Config.Config)) + scope, err := cgroup.Prepare(b.Conf.CgroupParentDir(), b.Conf.CgroupCPUFence(), spec.Rec.ID, cgroup.ResolveKnobs(&spec.Rec.Config.Config), spec.DeferCPUQuota) if err != nil { return 0, fmt.Errorf("prepare cgroup scope: %w", err) } @@ -167,6 +167,14 @@ func (b *Backend) LaunchVMProcess(ctx context.Context, spec LaunchSpec) (pid int return pid, nil } +// ArmCPUQuota sets the finite quota on a scope launched with DeferCPUQuota; call after memory load, before resume. +func (b *Backend) ArmCPUQuota(id string, cfg *types.Config) error { + if err := cgroup.Arm(b.Conf.CgroupParentDir(), id, cgroup.ResolveKnobs(cfg)); err != nil { + return fmt.Errorf("arm cpu quota: %w", err) + } + return nil +} + // AbortLaunch terminates a failed launch and clears runtime files. func (b *Backend) AbortLaunch(ctx context.Context, pid int, sockPath, runDir string, runtimeFiles []string) { _ = utils.TerminateProcess(ctx, pid, b.Conf.BinaryName(), sockPath, b.Conf.TerminateGracePeriod()) From 376087d237baca01858a6bd24ed6ef81ad74d885 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 5 Aug 2026 22:55:02 +0800 Subject: [PATCH 2/2] review: sink armQuota below the exported region An unexported helper sat between Arm and Remove; public-above-private puts it with the other control-file writers. --- cgroup/cgroup.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cgroup/cgroup.go b/cgroup/cgroup.go index a456b6e5..0e324f73 100644 --- a/cgroup/cgroup.go +++ b/cgroup/cgroup.go @@ -177,18 +177,6 @@ func Arm(parentDir, vmID string, k Knobs) error { return armQuota(ScopeDir(parentDir, vmID), k) } -func armQuota(dir string, k Knobs) error { - if err := writeControl(dir, maxName, fmt.Sprintf("%d %d", k.QuotaUs, k.PeriodUs)); err != nil { - return err - } - if k.BurstUs > 0 { - if err := writeControl(dir, burstName, strconv.FormatInt(k.BurstUs, 10)); err != nil { - return err - } - } - return nil -} - // Remove kills everything left in an owned scope and removes it; the VMM must already be confirmed dead. ENOENT counts as success. func Remove(ctx context.Context, parentDir, vmID string) error { dir := ScopeDir(parentDir, vmID) @@ -397,6 +385,18 @@ func enableControllers(dir string, ctrls []string) error { return writeControl(dir, subtreeControlName, strings.Join(missing, " ")) } +func armQuota(dir string, k Knobs) error { + if err := writeControl(dir, maxName, fmt.Sprintf("%d %d", k.QuotaUs, k.PeriodUs)); err != nil { + return err + } + if k.BurstUs > 0 { + if err := writeControl(dir, burstName, strconv.FormatInt(k.BurstUs, 10)); err != nil { + return err + } + } + return nil +} + func readControl(dir, name string) (string, error) { data, err := os.ReadFile(filepath.Join(dir, name)) //nolint:gosec // fixed name under the config-derived parent return strings.TrimSpace(string(data)), err