diff --git a/e2etests/MODULE.bazel b/e2etests/MODULE.bazel index b7c91966dda..11521274cea 100644 --- a/e2etests/MODULE.bazel +++ b/e2etests/MODULE.bazel @@ -20,3 +20,14 @@ local_path_override( module_name = "com_github_google_android_cuttlefish_frontend", path = "../frontend", ) + +# Expose the cuttlefish-host-resources init script (which lives in +# ../base/debian, outside this Bazel module) as a data dependency for the +# host_resources e2e tests. +new_local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") + +new_local_repository( + name = "cuttlefish_debian", + build_file_content = 'exports_files(glob(["**"]))\n', + path = "../base/debian", +) diff --git a/e2etests/host_resources/common/BUILD.bazel b/e2etests/host_resources/common/BUILD.bazel new file mode 100644 index 00000000000..4e036155b62 --- /dev/null +++ b/e2etests/host_resources/common/BUILD.bazel @@ -0,0 +1,33 @@ +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "common", + srcs = [ + "filesystem_ns.go", + "ip.go", + "network_ns.go", + "nft.go", + "sandbox.go", + "state.go", + ], + importpath = "github.com/google/android-cuttlefish/e2etests/host_resources/common", + visibility = ["//visibility:public"], + deps = [ + "@com_github_google_go_cmp//cmp", + "@rules_go//go/runfiles", + ], +) diff --git a/e2etests/host_resources/common/filesystem_ns.go b/e2etests/host_resources/common/filesystem_ns.go new file mode 100644 index 00000000000..a7924bf293c --- /dev/null +++ b/e2etests/host_resources/common/filesystem_ns.go @@ -0,0 +1,79 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +func (s *Sandbox) setupFilesystem() error { + nsPath := filepath.Join(s.tempdir, "nsswitch.conf") + if err := os.WriteFile(nsPath, []byte(constructSandboxNsswitch()), 0644); err != nil { + return fmt.Errorf("writing nsswitch override: %w", err) + } + grpPath := filepath.Join(s.tempdir, "group") + if err := os.WriteFile(grpPath, []byte(constructSandboxGroupFile()), 0644); err != nil { + return fmt.Errorf("writing group override: %w", err) + } + script := fmt.Sprintf( + "set -e; "+ + "mount --bind %q /etc/nsswitch.conf; "+ + "mount --bind %q /etc/group; "+ + "mount -t tmpfs tmpfs /etc/default; "+ + ": > /etc/default/cuttlefish-host-resources; "+ + "mount -t tmpfs tmpfs /run", + nsPath, grpPath) + _, err := s.Run("sh", "-c", script) + return err +} + +// reconstruct the host's nsswitch file to use only the group file +func constructSandboxNsswitch() string { + b, err := os.ReadFile("/etc/nsswitch.conf") + if err != nil { + return "passwd: files\ngroup: files\n" + } + lines := strings.Split(string(b), "\n") + found := false + for i, l := range lines { + if strings.HasPrefix(strings.TrimSpace(l), "group:") { + lines[i] = "group: files" + found = true + } + } + if !found { + lines = append(lines, "group: files") + } + return strings.Join(lines, "\n") + "\n" +} + + +// reconstruct the host's group file, but with cvdnetwork as gid 0, +// mostly for convenience +func constructSandboxGroupFile() string { + b, _ := os.ReadFile("/etc/group") + var out []string + for _, l := range strings.Split(string(b), "\n") { + if l == "" || strings.HasPrefix(l, "cvdnetwork:") { + continue + } + out = append(out, l) + } + out = append(out, "cvdnetwork:x:0:") + return strings.Join(out, "\n") + "\n" +} diff --git a/e2etests/host_resources/common/ip.go b/e2etests/host_resources/common/ip.go new file mode 100644 index 00000000000..af4118bc834 --- /dev/null +++ b/e2etests/host_resources/common/ip.go @@ -0,0 +1,102 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "encoding/json" + "fmt" +) + +// derived from `ip -details -json link` +type Link struct { + Ifindex int `json:"ifindex"` + Ifname string `json:"ifname"` + Flags []string `json:"flags"` + Operstate string `json:"operstate"` + Master string `json:"master"` + Address string `json:"address"` + LinkInfo *LinkInfo `json:"linkinfo"` +} + +type LinkInfo struct { + InfoKind string `json:"info_kind"` + InfoData *TunData `json:"info_data"` +} + +type TunData struct { + Type string `json:"type"` + VnetHdr bool `json:"vnet_hdr"` + Group string `json:"group"` +} + +func (l Link) Kind() string { + if l.LinkInfo != nil { + return l.LinkInfo.InfoKind + } + return "" +} + +func (l Link) IsUp() bool { + for _, f := range l.Flags { + if f == "UP" { + return true + } + } + return false +} + +func (l Link) VnetHdr() bool { + return l.LinkInfo != nil && l.LinkInfo.InfoData != nil && l.LinkInfo.InfoData.VnetHdr +} + +// derived from `ip -json addr`. +type Addr struct { + Ifname string `json:"ifname"` + AddrInfo []AddrInfo `json:"addr_info"` +} + +type AddrInfo struct { + Family string `json:"family"` + Local string `json:"local"` + Prefixlen int `json:"prefixlen"` + Scope string `json:"scope"` +} + +func parseLinks(s string) []Link { + var links []Link + json.Unmarshal([]byte(s), &links) + return links +} + +func parseAddrs(s string) []Addr { + var addrs []Addr + json.Unmarshal([]byte(s), &addrs) + return addrs +} + +// the first non-link-local IPv4 CIDR on the interface, or "" +func (hs HostState) PrimaryIPv4(ifname string) string { + for _, a := range hs.Addrs { + if a.Ifname != ifname { + continue + } + for _, ai := range a.AddrInfo { + if ai.Family == "inet" { + return fmt.Sprintf("%s/%d", ai.Local, ai.Prefixlen) + } + } + } + return "" +} diff --git a/e2etests/host_resources/common/network_ns.go b/e2etests/host_resources/common/network_ns.go new file mode 100644 index 00000000000..89b04a61813 --- /dev/null +++ b/e2etests/host_resources/common/network_ns.go @@ -0,0 +1,55 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "os" + "os/exec" + "strings" + "testing" +) + +func checkNetworkPrereqs(t *testing.T) { + t.Helper() + ensurePath() + for _, bin := range []string{"ip", "nft"} { + if _, err := exec.LookPath(bin); err != nil { + t.Fatalf("required binary %q not found on PATH: %v", bin, err) + } + } + if _, err := os.Stat("/dev/net/tun"); err != nil { + t.Fatalf("/dev/net/tun is not available: %v", err) + } +} + +// make sure the sbin directories that hold ip and nft are on PATH +func ensurePath() { + parts := strings.Split(os.Getenv("PATH"), ":") + have := map[string]bool{} + for _, p := range parts { + have[p] = true + } + for _, e := range []string{"/usr/sbin", "/usr/bin", "/sbin", "/bin"} { + if !have[e] { + parts = append(parts, e) + } + } + os.Setenv("PATH", strings.Join(parts, ":")) +} + +func (s *Sandbox) setupNetwork() error { + _, err := s.Run("ip", "link", "set", "lo", "up") + return err +} diff --git a/e2etests/host_resources/common/nft.go b/e2etests/host_resources/common/nft.go new file mode 100644 index 00000000000..27aa391895e --- /dev/null +++ b/e2etests/host_resources/common/nft.go @@ -0,0 +1,177 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "encoding/json" + "fmt" + "sort" +) + +type NftRuleset struct { + Tables []NftTable + Chains []NftChain + Rules []NftRule +} + +type NftTable struct { + Family string + Name string + Handle int +} + +func (t NftTable) less(o NftTable) bool { + if t.Family != o.Family { + return t.Family < o.Family + } + return t.Name < o.Name +} + +type NftChain struct { + Family string + Table string + Name string + Type string + Hook string + Handle int +} + +func (c NftChain) less(o NftChain) bool { + if c.Family != o.Family { + return c.Family < o.Family + } + if c.Table != o.Table { + return c.Table < o.Table + } + return c.Name < o.Name +} + +type NftRule struct { + Family string + Table string + Chain string + Handle int + Masquerade bool + SaddrPrefix string +} + +func (r NftRule) less(o NftRule) bool { + if r.Family != o.Family { + return r.Family < o.Family + } + if r.Table != o.Table { + return r.Table < o.Table + } + if r.Chain != o.Chain { + return r.Chain < o.Chain + } + if r.SaddrPrefix != o.SaddrPrefix { + return r.SaddrPrefix < o.SaddrPrefix + } + return !r.Masquerade && o.Masquerade +} + +func parseNftRuleset(s string) NftRuleset { + var top struct { + Nftables []map[string]json.RawMessage `json:"nftables"` + } + if err := json.Unmarshal([]byte(s), &top); err != nil { + return NftRuleset{} + } + var rs NftRuleset + for _, el := range top.Nftables { + if raw, ok := el["table"]; ok { + var t struct { + Family string `json:"family"` + Name string `json:"name"` + Handle int `json:"handle"` + } + if json.Unmarshal(raw, &t) == nil { + rs.Tables = append(rs.Tables, NftTable{Family: t.Family, Name: t.Name, Handle: t.Handle}) + } + } + if raw, ok := el["chain"]; ok { + var c struct { + Family string `json:"family"` + Table string `json:"table"` + Name string `json:"name"` + Type string `json:"type"` + Hook string `json:"hook"` + Handle int `json:"handle"` + } + if json.Unmarshal(raw, &c) == nil { + rs.Chains = append(rs.Chains, NftChain{Family: c.Family, Table: c.Table, Name: c.Name, Type: c.Type, Hook: c.Hook, Handle: c.Handle}) + } + } + if raw, ok := el["rule"]; ok { + rs.Rules = append(rs.Rules, parseNftRule(raw)) + } + } + return rs +} + +func parseNftRule(raw json.RawMessage) NftRule { + var r struct { + Family string `json:"family"` + Table string `json:"table"` + Chain string `json:"chain"` + Handle int `json:"handle"` + Expr []json.RawMessage `json:"expr"` + } + json.Unmarshal(raw, &r) + nr := NftRule{Family: r.Family, Table: r.Table, Chain: r.Chain, Handle: r.Handle} + for _, e := range r.Expr { + var m map[string]json.RawMessage + if json.Unmarshal(e, &m) != nil { + continue + } + if _, ok := m["masquerade"]; ok { + nr.Masquerade = true + } + if mraw, ok := m["match"]; ok { + var match struct { + Left struct { + Payload *struct { + Field string `json:"field"` + } `json:"payload"` + } `json:"left"` + Right json.RawMessage `json:"right"` + } + if json.Unmarshal(mraw, &match) == nil && match.Left.Payload != nil && match.Left.Payload.Field == "saddr" { + var pfx struct { + Prefix *struct { + Addr string `json:"addr"` + Len int `json:"len"` + } `json:"prefix"` + } + if json.Unmarshal(match.Right, &pfx) == nil && pfx.Prefix != nil { + nr.SaddrPrefix = fmt.Sprintf("%s/%d", pfx.Prefix.Addr, pfx.Prefix.Len) + } + } + } + } + return nr +} + +func (rs NftRuleset) MasqueradeSaddrs() []string { + var out []string + for _, r := range rs.Rules { + if r.Masquerade && r.SaddrPrefix != "" { + out = append(out, r.SaddrPrefix) + } + } + sort.Strings(out) + return out +} diff --git a/e2etests/host_resources/common/sandbox.go b/e2etests/host_resources/common/sandbox.go new file mode 100644 index 00000000000..1e65318b720 --- /dev/null +++ b/e2etests/host_resources/common/sandbox.go @@ -0,0 +1,167 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "bytes" + "fmt" + "log" + "os" + "os/exec" + "strconv" + "strings" + "testing" + "time" + + "github.com/bazelbuild/rules_go/go/runfiles" +) + +type CommandOutput struct { + Stdout string + Stderr string +} + +type Sandbox struct { + t *testing.T + keeper *exec.Cmd + pid int + tempdir string + initScript string + closed bool +} + +type InitEnv struct { + NumCvdAccounts int +} + +func NewSandbox(t *testing.T) *Sandbox { + t.Helper() + checkPrereqs(t) + + s := &Sandbox{t: t, tempdir: t.TempDir()} + + script := os.Getenv("INIT_SCRIPT") + if script == "" { + t.Fatal("INIT_SCRIPT env var is not set (expected the host-resources init script runfile)") + } + full, err := runfiles.Rlocation(script) + if err != nil { + t.Fatalf("failed to locate init script runfile %q: %v", script, err) + } + if _, err := os.Stat(full); err != nil { + t.Fatalf("init script %q does not exist: %v", full, err) + } + s.initScript = full + + // spawn the process that will keep the sandbox alive + cmd := exec.Command("unshare", "--user", "--map-root-user", "--net", "--mount", "sleep", "infinity") + if err := cmd.Start(); err != nil { + t.Fatalf("cannot create rootless user+net namespace: %v", err) + } + s.keeper = cmd + s.pid = cmd.Process.Pid + + if err := s.waitReady(); err != nil { + s.Close() + t.Fatalf("namespace not usable on this host: %v", err) + } + if err := s.setupFilesystem(); err != nil { + s.Close() + t.Fatalf("failed to prepare filesystem sandbox: %v", err) + } + if err := s.setupNetwork(); err != nil { + s.Close() + t.Fatalf("failed to prepare network sandbox: %v", err) + } + + t.Cleanup(s.Close) + return s +} + +func checkPrereqs(t *testing.T) { + t.Helper() + checkNamespacePrereqs(t) + checkNetworkPrereqs(t) +} + +func checkNamespacePrereqs(t *testing.T) { + t.Helper() + if b, err := os.ReadFile("/proc/sys/kernel/unprivileged_userns_clone"); err == nil { + if strings.TrimSpace(string(b)) == "0" { + t.Fatal("unprivileged user namespaces are disabled (unprivileged_userns_clone=0)") + } + } + for _, bin := range []string{"unshare", "nsenter", "sleep"} { + if _, err := exec.LookPath(bin); err != nil { + t.Fatalf("required binary %q not found on PATH: %v", bin, err) + } + } +} + +func (s *Sandbox) waitReady() error { + deadline := time.Now().Add(5 * time.Second) + var lastErr error + for time.Now().Before(deadline) { + c := exec.Command("nsenter", "-t", strconv.Itoa(s.pid), "-U", "-n", "-m", "--preserve-credentials", "--", "true") + if err := c.Run(); err == nil { + return nil + } else { + lastErr = err + } + time.Sleep(50 * time.Millisecond) + } + return fmt.Errorf("namespace did not become ready: %w", lastErr) +} + +func (s *Sandbox) nsenterArgs(extra ...string) []string { + base := []string{"nsenter", "-t", strconv.Itoa(s.pid), "-U", "-n", "-m", "--preserve-credentials", "--"} + return append(base, extra...) +} + +func (s *Sandbox) Run(args ...string) (CommandOutput, error) { + full := s.nsenterArgs(args...) + cmd := exec.Command(full[0], full[1:]...) + var outBuf, errBuf bytes.Buffer + cmd.Stdout = &outBuf + cmd.Stderr = &errBuf + log.Printf("[sandbox] running: %s", strings.Join(args, " ")) + err := cmd.Run() + out := CommandOutput{Stdout: outBuf.String(), Stderr: errBuf.String()} + if err != nil { + return out, fmt.Errorf("command %q failed: %w (stderr: %s)", strings.Join(args, " "), err, strings.TrimSpace(errBuf.String())) + } + return out, nil +} + +func (s *Sandbox) RunHostResourcesInit(action string, ie InitEnv) error { + args := []string{"env"} + if ie.NumCvdAccounts > 0 { + args = append(args, fmt.Sprintf("num_cvd_accounts=%d", ie.NumCvdAccounts)) + } + args = append(args, "sh", s.initScript, action) + _, err := s.Run(args...) + return err +} + +func (s *Sandbox) Close() { + if s.closed { + return + } + s.closed = true + if s.keeper != nil && s.keeper.Process != nil { + s.keeper.Process.Kill() + s.keeper.Wait() + } +} diff --git a/e2etests/host_resources/common/state.go b/e2etests/host_resources/common/state.go new file mode 100644 index 00000000000..a7250e0565b --- /dev/null +++ b/e2etests/host_resources/common/state.go @@ -0,0 +1,154 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "path/filepath" + "sort" + "strings" + + "github.com/google/go-cmp/cmp" +) + +type HostState struct { + Nft NftRuleset + Links []Link + Addrs []Addr + Sysctls map[string]string +} + +func (s *Sandbox) Snapshot() HostState { + hs := HostState{Sysctls: map[string]string{}} + + if out, err := s.Run("nft", "-j", "list", "ruleset"); err == nil { + hs.Nft = parseNftRuleset(out.Stdout) + } else { + s.t.Logf("snapshot: nft list failed: %v", err) + } + if out, err := s.Run("ip", "-details", "-json", "link"); err == nil { + hs.Links = parseLinks(out.Stdout) + } else { + s.t.Logf("snapshot: ip link failed: %v", err) + } + if out, err := s.Run("ip", "-json", "addr"); err == nil { + hs.Addrs = parseAddrs(out.Stdout) + } else { + s.t.Logf("snapshot: ip addr failed: %v", err) + } + for key, path := range map[string]string{ + "net.ipv4.ip_forward": "/proc/sys/net/ipv4/ip_forward", + "net.ipv6.conf.all.forwarding": "/proc/sys/net/ipv6/conf/all/forwarding", + } { + if out, err := s.Run("cat", path); err == nil { + hs.Sysctls[key] = strings.TrimSpace(out.Stdout) + } + } + return hs +} + +// We can only check for the pidfiles, so we just do that to make sure +// dnsmasq was invoked. +func (s *Sandbox) DnsmasqPidfileIfaces() []string { + out, err := s.Run("sh", "-c", `ls -1 /run/cuttlefish-dnsmasq-*.pid 2>/dev/null || true`) + if err != nil { + return nil + } + var ifaces []string + for _, line := range nonEmptyLines(out.Stdout) { + base := filepath.Base(line) + ifaces = append(ifaces, strings.TrimSuffix(strings.TrimPrefix(base, "cuttlefish-dnsmasq-"), ".pid")) + } + sort.Strings(ifaces) + return ifaces +} + +func (s *Sandbox) HandleFiles() []string { + out, err := s.Run("sh", "-c", `ls -1 /run/cuttlefish/ 2>/dev/null || true`) + if err != nil { + return nil + } + files := nonEmptyLines(out.Stdout) + sort.Strings(files) + return files +} + +func DiffState(a, b HostState) string { + return cmp.Diff(a, b) +} + +// Normalize strips volatile fields (nft handles, ifindex, MAC, link-local IPv6) +// so two snapshots can be compared for leaks. +func Normalize(hs HostState) HostState { + out := HostState{} + + nft := NftRuleset{} + nft.Tables = append(nft.Tables, hs.Nft.Tables...) + nft.Chains = append(nft.Chains, hs.Nft.Chains...) + nft.Rules = append(nft.Rules, hs.Nft.Rules...) + for i := range nft.Tables { + nft.Tables[i].Handle = 0 + } + for i := range nft.Chains { + nft.Chains[i].Handle = 0 + } + for i := range nft.Rules { + nft.Rules[i].Handle = 0 + } + sort.Slice(nft.Tables, func(i, j int) bool { return nft.Tables[i].less(nft.Tables[j]) }) + sort.Slice(nft.Chains, func(i, j int) bool { return nft.Chains[i].less(nft.Chains[j]) }) + sort.Slice(nft.Rules, func(i, j int) bool { return nft.Rules[i].less(nft.Rules[j]) }) + out.Nft = nft + + for _, l := range hs.Links { + nl := Link{Ifname: l.Ifname, Master: l.Master} + if l.IsUp() { + nl.Flags = []string{"UP"} + } + if l.LinkInfo != nil { + nl.LinkInfo = &LinkInfo{InfoKind: l.LinkInfo.InfoKind} + if l.LinkInfo.InfoData != nil { + nl.LinkInfo.InfoData = &TunData{Type: l.LinkInfo.InfoData.Type, VnetHdr: l.LinkInfo.InfoData.VnetHdr} + } + } + out.Links = append(out.Links, nl) + } + sort.Slice(out.Links, func(i, j int) bool { return out.Links[i].Ifname < out.Links[j].Ifname }) + + for _, a := range hs.Addrs { + na := Addr{Ifname: a.Ifname} + for _, ai := range a.AddrInfo { + if ai.Family == "inet6" && strings.HasPrefix(ai.Local, "fe80") { + continue + } + na.AddrInfo = append(na.AddrInfo, AddrInfo{Family: ai.Family, Local: ai.Local, Prefixlen: ai.Prefixlen}) + } + sort.Slice(na.AddrInfo, func(i, j int) bool { return na.AddrInfo[i].Local < na.AddrInfo[j].Local }) + out.Addrs = append(out.Addrs, na) + } + sort.Slice(out.Addrs, func(i, j int) bool { return out.Addrs[i].Ifname < out.Addrs[j].Ifname }) + + return out +} + +func nonEmptyLines(s string) []string { + var out []string + for _, l := range strings.Split(s, "\n") { + l = strings.TrimSpace(l) + if l != "" { + out = append(out, l) + } + } + return out +} diff --git a/e2etests/host_resources/static_resources_init_test/BUILD.bazel b/e2etests/host_resources/static_resources_init_test/BUILD.bazel new file mode 100644 index 00000000000..6ba1a4021c7 --- /dev/null +++ b/e2etests/host_resources/static_resources_init_test/BUILD.bazel @@ -0,0 +1,38 @@ +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "static_resources_init_test", + size = "large", + srcs = ["main_test.go"], + data = [ + "@cuttlefish_debian//:cuttlefish-base.cuttlefish-host-resources.init", + ], + env = { + "INIT_SCRIPT": "$(rlocationpath @cuttlefish_debian//:cuttlefish-base.cuttlefish-host-resources.init)", + }, + tags = [ + "exclusive", + "external", + "local", + "no-sandbox", + ], + deps = [ + "//host_resources/common", + "@com_github_google_go_cmp//cmp", + "@com_github_google_go_cmp//cmp/cmpopts", + ], +) diff --git a/e2etests/host_resources/static_resources_init_test/main_test.go b/e2etests/host_resources/static_resources_init_test/main_test.go new file mode 100644 index 00000000000..22b3af5b0cc --- /dev/null +++ b/e2etests/host_resources/static_resources_init_test/main_test.go @@ -0,0 +1,151 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/google/android-cuttlefish/e2etests/host_resources/common" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +type netIface struct { + Name string + Addr string // primary IPv4 CIDR, "" if none + Master string // bridge it is enslaved to, "" if none + Tun bool // tap opened with vnet_hdr +} + +// startState is the full network state expected after `start`. +type startState struct { + Ifaces []netIface + NftTables []common.NftTable + NftChains []common.NftChain + Masquerades []string + Sysctls map[string]string + HandleFiles []string + DnsmasqIfaces []string +} + +func TestStaticResourcesInit(t *testing.T) { + cases := []struct { + name string + env common.InitEnv + want startState + }{ + { + name: "num_cvd_accounts=1", + env: common.InitEnv{NumCvdAccounts: 1}, + want: startState{ + Ifaces: []netIface{ + {Name: "cvd-ebr", Addr: "192.168.98.1/24"}, + {Name: "cvd-wbr", Addr: "192.168.96.1/24"}, + {Name: "cvd-etap-01", Master: "cvd-ebr", Tun: true}, + {Name: "cvd-wtap-01", Master: "cvd-wbr", Tun: true}, + {Name: "cvd-mtap-01", Addr: "192.168.97.1/30", Tun: true}, + {Name: "cvd-wifiap-01", Addr: "192.168.94.1/30", Tun: true}, + }, + NftTables: []common.NftTable{ + {Family: "ip", Name: "cuttlefish_nat"}, + {Family: "bridge", Name: "cuttlefish_bridge"}, + }, + NftChains: []common.NftChain{ + {Family: "ip", Table: "cuttlefish_nat", Name: "postrouting", Type: "nat", Hook: "postrouting"}, + {Family: "bridge", Table: "cuttlefish_bridge", Name: "prerouting", Type: "filter", Hook: "prerouting"}, + {Family: "bridge", Table: "cuttlefish_bridge", Name: "forward", Type: "filter", Hook: "forward"}, + }, + Masquerades: []string{"192.168.94.0/30", "192.168.96.0/24", "192.168.97.0/30", "192.168.98.0/24"}, + Sysctls: map[string]string{"net.ipv4.ip_forward": "1", "net.ipv6.conf.all.forwarding": "1"}, + HandleFiles: []string{"masq-br-cvd-ebr.handle", "masq-br-cvd-wbr.handle", "masq-cvd-mtap-01.handle", "masq-cvd-wifiap-01.handle"}, + DnsmasqIfaces: []string{"cvd-ebr", "cvd-wbr"}, + }, + }, + } + + opts := cmp.Options{ + cmpopts.SortSlices(func(a, b netIface) bool { return a.Name < b.Name }), + cmpopts.SortSlices(func(a, b common.NftTable) bool { + if a.Family != b.Family { + return a.Family < b.Family + } + return a.Name < b.Name + }), + cmpopts.SortSlices(func(a, b common.NftChain) bool { + if a.Family != b.Family { + return a.Family < b.Family + } + if a.Table != b.Table { + return a.Table < b.Table + } + return a.Name < b.Name + }), + cmpopts.IgnoreFields(common.NftTable{}, "Handle"), + cmpopts.IgnoreFields(common.NftChain{}, "Handle"), + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := common.NewSandbox(t) + defer s.Close() + + base := s.Snapshot() + + if err := s.RunHostResourcesInit("start", tc.env); err != nil { + t.Fatalf("start: %v", err) + } + + if diff := cmp.Diff(tc.want, observe(s), opts); diff != "" { + t.Errorf("host state after start (-want +got):\n%s", diff) + } + + if err := s.RunHostResourcesInit("stop", tc.env); err != nil { + t.Fatalf("stop: %v", err) + } + afterStop := s.Snapshot() + if diff := common.DiffState(common.Normalize(base), common.Normalize(afterStop)); diff != "" { + t.Errorf("state leaked after stop (-before +after):\n%s", diff) + } + + if diff := cmp.Diff(tc.want.Sysctls, afterStop.Sysctls); diff != "" { + t.Errorf("forwarding sysctls changed after stop (-want +got):\n%s", diff) + } + }) + } +} + +func observe(s *common.Sandbox) startState { + hs := s.Snapshot() + got := startState{ + NftTables: hs.Nft.Tables, + NftChains: hs.Nft.Chains, + Masquerades: hs.Nft.MasqueradeSaddrs(), + Sysctls: hs.Sysctls, + HandleFiles: s.HandleFiles(), + DnsmasqIfaces: s.DnsmasqPidfileIfaces(), + } + for _, l := range hs.Links { + if l.Ifname == "lo" { + continue + } + got.Ifaces = append(got.Ifaces, netIface{ + Name: l.Ifname, + Addr: hs.PrimaryIPv4(l.Ifname), + Master: l.Master, + Tun: l.Kind() == "tun" && l.VnetHdr(), + }) + } + return got +}