vm: exempt clone/restore provisioning from the child VM's CPU ceiling (#186)

cgroup/cgroup.go

@@ -133,8 +133,8 @@ func ScopeDir(parentDir, vmID string) string {
 	return filepath.Join(parentDir, scopePrefix+vmID+scopeSuffix)
 }
 
-// Prepare creates or reconfigures vmID's scope and returns its opened directory for CLONE_INTO_CGROUP; idempotent, so a relaunch reuses a scope its dying predecessor still occupies.
-func Prepare(parentDir, fence, vmID string, k Knobs) (*os.File, error) {
+// Prepare creates or reconfigures vmID's scope and returns its opened directory for CLONE_INTO_CGROUP; idempotent, so a relaunch reuses a scope its dying predecessor still occupies. deferQuota leaves the ceiling at max for the provisioning window — weight, fence, and placement still apply — until Arm sets the finite quota.
+func Prepare(parentDir, fence, vmID string, k Knobs, deferQuota bool) (*os.File, error) {
 	if vmID == "" {
 		return nil, errors.New("cgroup scope: empty vm id")
 	}
@@ -158,13 +158,12 @@ func Prepare(parentDir, fence, vmID string, k Knobs) (*os.File, error) {
 			return nil, err
 		}
 	}
-	if err := writeControl(dir, maxName, fmt.Sprintf("%d %d", k.QuotaUs, k.PeriodUs)); err != nil {
-		return nil, err
-	}
-	if k.BurstUs > 0 {
-		if err := writeControl(dir, burstName, strconv.FormatInt(k.BurstUs, 10)); err != nil {
+	if deferQuota {
+		if err := writeControl(dir, maxName, fmt.Sprintf("max %d", k.PeriodUs)); err != nil {
 			return nil, err
 		}
+	} else if err := armQuota(dir, k); err != nil {
+		return nil, err
 	}
 	scope, err := os.Open(dir) //nolint:gosec // path derives from config parent + generated VM ID
 	if err != nil {
@@ -173,6 +172,11 @@ func Prepare(parentDir, fence, vmID string, k Knobs) (*os.File, error) {
 	return scope, nil
 }
 
+// Arm sets the finite quota on a scope prepared with deferQuota; called after memory load, before resume, so the guest never runs uncapped. Idempotent — a retry after a partial failure converges.
+func Arm(parentDir, vmID string, k Knobs) error {
+	return armQuota(ScopeDir(parentDir, vmID), k)
+}
+
 // Remove kills everything left in an owned scope and removes it; the VMM must already be confirmed dead. ENOENT counts as success.
 func Remove(ctx context.Context, parentDir, vmID string) error {
 	dir := ScopeDir(parentDir, vmID)
@@ -381,6 +385,18 @@ func enableControllers(dir string, ctrls []string) error {
 	return writeControl(dir, subtreeControlName, strings.Join(missing, " "))
 }
 
+func armQuota(dir string, k Knobs) error {
+	if err := writeControl(dir, maxName, fmt.Sprintf("%d %d", k.QuotaUs, k.PeriodUs)); err != nil {
+		return err
+	}
+	if k.BurstUs > 0 {
+		if err := writeControl(dir, burstName, strconv.FormatInt(k.BurstUs, 10)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func readControl(dir, name string) (string, error) {
 	data, err := os.ReadFile(filepath.Join(dir, name)) //nolint:gosec // fixed name under the config-derived parent
 	return strings.TrimSpace(string(data)), err

cgroup/cgroup_test.go

@@ -235,3 +235,24 @@ func TestEffectiveCPUs(t *testing.T) {
 		t.Error("no constraint: want nil")
 	}
 }
+
+func TestArmSetsQuotaThenBurst(t *testing.T) {
+	parent := t.TempDir()
+	dir := ScopeDir(parent, "A")
+	if err := os.Mkdir(dir, 0o755); err != nil {
+		t.Fatalf("setup: %v", err)
+	}
+	for _, f := range []string{"cpu.max", "cpu.max.burst"} {
+		if err := os.WriteFile(filepath.Join(dir, f), nil, 0o600); err != nil {
+			t.Fatalf("setup: %v", err)
+		}
+	}
+	if err := Arm(parent, "A", Knobs{QuotaUs: 150000, PeriodUs: 100000, BurstUs: 50000}); err != nil {
+		t.Fatalf("Arm: %v", err)
+	}
+	max, _ := os.ReadFile(filepath.Join(dir, "cpu.max"))
+	burst, _ := os.ReadFile(filepath.Join(dir, "cpu.max.burst"))
+	if string(max) != "150000 100000" || string(burst) != "50000" {
+		t.Errorf("max=%q burst=%q", max, burst)
+	}
+}

docs/vm.md

@@ -37,6 +37,8 @@ Defaults are Kubernetes-style Guaranteed at N for `--cpu N`: quota = N cores (`-
 
 `cgroup_cpus` fences the whole VM population onto a host cpu subset (e.g. `0-14` on a 16-core host keeps core 15 for the OS, the API consumer, and clone/wake execution). `--cpuset-cpus` pins one VM to specific cores inside the fence. Both are validated by cocoon against the effective sets — the kernel silently degrades ungrantable cpuset requests rather than failing — and shrinking the fence is refused while a running VM's placement conflicts. Clearing `cgroup_cpus` converges: the stale fence is reset on the next launch.
 
+Provisioning is exempt from the ceiling: clone/restore launch with the quota at `max` — weight, fence, and placement still apply — and the finite quota is armed after the memory load completes, before the guest resumes, so snapshot loading runs at control-plane speed while the guest never executes uncapped. The mmap restore mode's deferred first-touch faults are inherent guest-lifetime work and do pay the VM's quota — on dense hosts prefer `copy` mode for latency-critical wakes or grant quota headroom.
+
 cgroup knobs are host-side policy, like networking: snapshots record the source VM's values but never apply them — a clone takes its policy from flags (defaults otherwise), restore keeps the target VM's. Scopes are removed when the VMM dies (stop, hibernate, delete, crash convergence) and orphans are swept by `cocoon gc`. `cocoon vm list` shows per-VM throttling as `THROTTLED` (`nr_throttled/throttled_usec` from `cpu.stat`).
 
 Requirements: cgroup v2 unified hierarchy with the `cpu` controller (kernel ≥ 5.14 for burst), running cocoon as root (production shape). Non-root works inside a systemd user slice with delegated controllers (`systemd-run --user --scope`), where user slices typically delegate `cpu` but not `cpuset` — fence/placement then fail preflight with the exact missing file named.

hypervisor/backend.go

@@ -106,6 +106,9 @@ type LaunchSpec struct {
 
 	// Rec names the VM whose CPU scope the process enters at spawn (ID + cgroup knobs); every VM enters a scope.
 	Rec *VMRecord
+
+	// DeferCPUQuota leaves the scope's ceiling at max through the paused provisioning window (#186); the backend arms the finite quota before resume.
+	DeferCPUQuota bool
 }
 
 // PreflightHook validates rec against the snapshot source dir before anything is applied.

hypervisor/cloudhypervisor/clone.go

@@ -22,6 +22,7 @@ import (
 const restoreTAPPrefix = "rm"
 
 type cloneResumeOpts struct {
+	vmID                string
 	vmCfg               *types.VMConfig
 	directBoot          bool
 	hadCidataInSnapshot bool
@@ -130,13 +131,14 @@ func (ch *CloudHypervisor) cloneAfterExtractParsed(ctx context.Context, vmID str
 		VM:     types.VM{ID: vmID, Config: *vmCfg},
 		RunDir: runDir,
 		LogDir: logDir,
-	}, args, net.NetnsPath)
+	}, args, net.NetnsPath, true)
 	if err != nil {
 		ch.MarkError(ctx, vmID)
 		return nil, fmt.Errorf("launch CH: %w", err)
 	}
 
 	if err := ch.restoreAndResumeClone(ctx, pid, sockPath, runDir, &cloneResumeOpts{
+		vmID:                vmID,
 		vmCfg:               vmCfg,
 		directBoot:          directBoot,
 		hadCidataInSnapshot: hadCidataInSnapshot,
@@ -198,6 +200,9 @@ func (ch *CloudHypervisor) restoreAndResumeClone(ctx context.Context, pid int, s
 			return fmt.Errorf("vm.add-disk (data %s): %w", sc.Serial, err)
 		}
 	}
+	if err = ch.ArmCPUQuota(opts.vmID, &opts.vmCfg.Config); err != nil {
+		return err
+	}
 	if err = resumeVM(ctx, hc); err != nil {
 		return fmt.Errorf("vm.resume: %w", err)
 	}

hypervisor/cloudhypervisor/clone_test.go

@@ -363,8 +363,16 @@ func TestRestoreAndResumeCloneHotplugsCidataByRole(t *testing.T) {
 		{Path: "/run/cidata.img", RO: true, Role: types.StorageRoleCidata},
 		{Path: "/run/extra.raw", Role: types.StorageRoleData, Serial: "extra"},
 	}
-	ch := &CloudHypervisor{conf: NewConfig(&config.Config{})}
+	scopeParent := t.TempDir()
+	if err := os.Mkdir(filepath.Join(scopeParent, "vm-T.scope"), 0o755); err != nil {
+		t.Fatalf("setup: %v", err)
+	}
+	ch := &CloudHypervisor{
+		Backend: &hypervisor.Backend{Conf: tmpScopeConf{Config: NewConfig(&config.Config{}), parent: scopeParent}},
+		conf:    NewConfig(&config.Config{}),
+	}
 	if err := ch.restoreAndResumeClone(t.Context(), 0, sock, t.TempDir(), &cloneResumeOpts{
+		vmID:           "T",
 		vmCfg:          &types.VMConfig{Config: types.Config{CPU: 2}},
 		storageConfigs: storageConfigs,
 		dataDisks:      storageConfigs[2:],
@@ -495,3 +503,11 @@ func basePatchOpts() *patchOptions {
 		directBoot:  true,
 	}
 }
+
+// tmpScopeConf redirects the cgroup parent to a temp dir so Arm writes plain files.
+type tmpScopeConf struct {
+	*Config
+	parent string
+}
+
+func (c tmpScopeConf) CgroupParentDir() string { return c.parent }

hypervisor/cloudhypervisor/restore.go

@@ -96,7 +96,7 @@ func (ch *CloudHypervisor) restoreAfterExtract(ctx context.Context, vmID string,
 
 	// Launch under the target config: a --force cross-config restore changes the vCPU count the scope derives from.
 	rec.Config = *vmCfg
-	pid, launchErr := ch.launchProcess(ctx, rec, args, rec.ResolvedNetnsPath())
+	pid, launchErr := ch.launchProcess(ctx, rec, args, rec.ResolvedNetnsPath(), true)
 	if launchErr != nil {
 		return nil, fmt.Errorf("launch CH: %w", launchErr)
 	}
@@ -112,6 +112,9 @@ func (ch *CloudHypervisor) restoreAfterExtract(ctx context.Context, vmID string,
 	if err = restoreVM(ctx, hc, rec.RunDir, vmCfg.RestoreMode); err != nil {
 		return nil, fmt.Errorf("vm.restore: %w", err)
 	}
+	if err = ch.ArmCPUQuota(vmID, &vmCfg.Config); err != nil {
+		return nil, err
+	}
 	if err = resumeVM(ctx, hc); err != nil {
 		return nil, fmt.Errorf("vm.resume: %w", err)
 	}

hypervisor/cloudhypervisor/start.go

@@ -23,12 +23,12 @@ func (ch *CloudHypervisor) startOne(ctx context.Context, id string) error {
 			vmCfg := buildVMConfig(rec, hypervisor.ConsoleSockPath(rec.RunDir), cgroup.EffectiveCPUs(rec.Config.CPUSetCPUs, ch.conf.CgroupCPUs))
 			args := buildCLIArgs(vmCfg, sockPath)
 			ch.saveCmdline(ctx, rec, args)
-			return ch.launchProcess(ctx, rec, args, rec.ResolvedNetnsPath())
+			return ch.launchProcess(ctx, rec, args, rec.ResolvedNetnsPath(), false)
 		},
 	})
 }
 
-func (ch *CloudHypervisor) launchProcess(ctx context.Context, rec *hypervisor.VMRecord, args []string, netnsPath string) (int, error) {
+func (ch *CloudHypervisor) launchProcess(ctx context.Context, rec *hypervisor.VMRecord, args []string, netnsPath string, deferQuota bool) (int, error) {
 	processLog := ch.LogFilePath(rec.LogDir)
 	logFile, err := os.Create(processLog) //nolint:gosec
 	if err != nil {
@@ -47,9 +47,10 @@ func (ch *CloudHypervisor) launchProcess(ctx context.Context, rec *hypervisor.VM
 	}
 
 	pid, err := ch.LaunchVMProcess(ctx, hypervisor.LaunchSpec{
-		Cmd:       cmd,
-		NetnsPath: netnsPath,
-		Rec:       rec,
+		Cmd:           cmd,
+		NetnsPath:     netnsPath,
+		Rec:           rec,
+		DeferCPUQuota: deferQuota,
 	})
 	if err != nil {
 		return 0, err

hypervisor/firecracker/clone.go

@@ -49,6 +49,8 @@ type launchCloneFn func([]*os.File) (int, *cloneLeaseControl, error)
 type cloneLaunch struct {
 	launch           launchCloneFn
 	sockPath, runDir string
+	vmID             string
+	vmCfg            *types.VMConfig
 	networkConfigs   []*types.NetworkConfig
 	src, dst         []*types.StorageConfig
 }
@@ -101,10 +103,10 @@ func (fc *Firecracker) cloneAfterExtract(ctx context.Context, vmID string, vmCfg
 			VM:     types.VM{ID: vmID, Config: *vmCfg},
 			RunDir: runDir,
 			LogDir: logDir,
-		}, sockPath, net.NetnsPath, leaseFiles)
+		}, sockPath, net.NetnsPath, leaseFiles, true)
 	}
 	pid, leaseControl, plan, cloneErr := fc.startCloneVM(ctx, cloneLaunch{
-		launch: launch, sockPath: sockPath, runDir: runDir,
+		launch: launch, sockPath: sockPath, runDir: runDir, vmID: vmID, vmCfg: vmCfg,
 		networkConfigs: networkConfigs, src: meta.StorageConfigs, dst: storageConfigs,
 	})
 	if cloneErr != nil {
@@ -200,6 +202,9 @@ func (fc *Firecracker) resumeAndReanchorClone(ctx context.Context, pid int, cl c
 		}
 	}()
 
+	if err = fc.ArmCPUQuota(cl.vmID, &cl.vmCfg.Config); err != nil {
+		return err
+	}
 	hc := utils.NewSocketHTTPClient(cl.sockPath)
 	if err = resumeVM(ctx, hc); err != nil {
 		return fmt.Errorf("resume: %w", err)

hypervisor/firecracker/restore.go

@@ -63,7 +63,7 @@ func (fc *Firecracker) restoreAfterExtract(ctx context.Context, vmID string, vmC
 
 	// Launch under the target config: a --force cross-config restore changes the vCPU count the scope derives from.
 	rec.Config = *vmCfg
-	pid, launchErr := fc.launchProcess(ctx, rec, sockPath, rec.ResolvedNetnsPath())
+	pid, launchErr := fc.launchProcess(ctx, rec, sockPath, rec.ResolvedNetnsPath(), true)
 	if launchErr != nil {
 		return nil, fmt.Errorf("launch FC: %w", launchErr)
 	}
@@ -79,6 +79,9 @@ func (fc *Firecracker) restoreAfterExtract(ctx context.Context, vmID string, vmC
 		return nil, fmt.Errorf("snapshot/load: %w", err)
 	}
 
+	if err = fc.ArmCPUQuota(vmID, &vmCfg.Config); err != nil {
+		return nil, err
+	}
 	hc := utils.NewSocketHTTPClient(sockPath)
 	if err = resumeVM(ctx, hc); err != nil {
 		return nil, fmt.Errorf("resume: %w", err)

hypervisor/firecracker/start.go

@@ -32,7 +32,7 @@ func (fc *Firecracker) startOne(ctx context.Context, id string) error {
 	return fc.StartSequence(ctx, id, hypervisor.StartSpec{
 		RuntimeFiles: runtimeFiles,
 		Launch: func(ctx context.Context, rec *hypervisor.VMRecord, sockPath string) (int, error) {
-			return fc.launchProcess(ctx, rec, sockPath, rec.ResolvedNetnsPath())
+			return fc.launchProcess(ctx, rec, sockPath, rec.ResolvedNetnsPath(), false)
 		},
 		PostLaunch: func(ctx context.Context, rec *hypervisor.VMRecord, sockPath string, _ int) error {
 			return fc.configureVM(ctx, utils.NewSocketHTTPClient(sockPath), rec)
@@ -149,13 +149,13 @@ func (c *cloneLeaseControl) commit() error {
 	return waitRelayLeaseResponse(c.responses, relayLeaseCommitAck, "commit")
 }
 
-func (fc *Firecracker) launchProcess(ctx context.Context, rec *hypervisor.VMRecord, sockPath, netnsPath string) (int, error) {
-	pid, _, err := fc.launchProcessWithLeases(ctx, rec, sockPath, netnsPath, nil)
+func (fc *Firecracker) launchProcess(ctx context.Context, rec *hypervisor.VMRecord, sockPath, netnsPath string, deferQuota bool) (int, error) {
+	pid, _, err := fc.launchProcessWithLeases(ctx, rec, sockPath, netnsPath, nil, deferQuota)
 	return pid, err
 }
 
 // launchProcessWithLeases keeps inherited source-VM leases in the console relay until the caller confirms clone setup.
-func (fc *Firecracker) launchProcessWithLeases(ctx context.Context, rec *hypervisor.VMRecord, sockPath, netnsPath string, leaseFiles []*os.File) (int, *cloneLeaseControl, error) {
+func (fc *Firecracker) launchProcessWithLeases(ctx context.Context, rec *hypervisor.VMRecord, sockPath, netnsPath string, leaseFiles []*os.File, deferQuota bool) (int, *cloneLeaseControl, error) {
 	logger := log.WithFunc("firecracker.launchProcessWithLeases")
 
 	fcLog := fc.LogFilePath(rec.LogDir)
@@ -183,10 +183,11 @@ func (fc *Firecracker) launchProcessWithLeases(ctx context.Context, rec *hypervi
 	fcCmd.Stdin = slave
 	fcCmd.Stdout = slave
 	pid, err := fc.LaunchVMProcess(ctx, hypervisor.LaunchSpec{
-		Cmd:       fcCmd,
-		NetnsPath: netnsPath,
-		OnFail:    func() { _ = master.Close() },
-		Rec:       rec,
+		Cmd:           fcCmd,
+		NetnsPath:     netnsPath,
+		OnFail:        func() { _ = master.Close() },
+		Rec:           rec,
+		DeferCPUQuota: deferQuota,
 	})
 	if err != nil {
 		return 0, nil, err

hypervisor/start.go

@@ -134,7 +134,7 @@ func (b *Backend) LaunchVMProcess(ctx context.Context, spec LaunchSpec) (pid int
 		}
 	}()
 
-	scope, err := cgroup.Prepare(b.Conf.CgroupParentDir(), b.Conf.CgroupCPUFence(), spec.Rec.ID, cgroup.ResolveKnobs(&spec.Rec.Config.Config))
+	scope, err := cgroup.Prepare(b.Conf.CgroupParentDir(), b.Conf.CgroupCPUFence(), spec.Rec.ID, cgroup.ResolveKnobs(&spec.Rec.Config.Config), spec.DeferCPUQuota)
 	if err != nil {
 		return 0, fmt.Errorf("prepare cgroup scope: %w", err)
 	}
@@ -167,6 +167,14 @@ func (b *Backend) LaunchVMProcess(ctx context.Context, spec LaunchSpec) (pid int
 	return pid, nil
 }
 
+// ArmCPUQuota sets the finite quota on a scope launched with DeferCPUQuota; call after memory load, before resume.
+func (b *Backend) ArmCPUQuota(id string, cfg *types.Config) error {
+	if err := cgroup.Arm(b.Conf.CgroupParentDir(), id, cgroup.ResolveKnobs(cfg)); err != nil {
+		return fmt.Errorf("arm cpu quota: %w", err)
+	}
+	return nil
+}
+
 // AbortLaunch terminates a failed launch and clears runtime files.
 func (b *Backend) AbortLaunch(ctx context.Context, pid int, sockPath, runDir string, runtimeFiles []string) {
 	_ = utils.TerminateProcess(ctx, pid, b.Conf.BinaryName(), sockPath, b.Conf.TerminateGracePeriod())