diff --git a/traceroute/fuzz_linux_test.go b/traceroute/fuzz_linux_test.go new file mode 100644 index 00000000..d1e9beda --- /dev/null +++ b/traceroute/fuzz_linux_test.go @@ -0,0 +1,33 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build linux + +package traceroute + +import ( + "net" + "testing" + "time" + + "golang.org/x/net/ipv4" +) + +func FuzzNetworkReplyParsers(f *testing.F) { + f.Add([]byte{}) + f.Add(make([]byte, ipv4.HeaderLen)) + f.Add(make([]byte, 128)) + target := net.ParseIP("203.0.113.10").To4() + local := net.ParseIP("192.0.2.20").To4() + peer := &net.IPAddr{IP: net.ParseIP("192.0.2.1")} + + f.Fuzz(func(_ *testing.T, packet []byte) { + parseUDPICMPReply(packet, peer, target, 31000, 33434, 9, time.Millisecond) + parseTCPICMPReply(packet, peer, target, 31000, 443, 0x12345678, time.Millisecond) + matchesICMPProbe(packet, peer.IP, target, 17, 23) + matchesTCPResponse(&ipv4.Header{Src: target, Dst: local}, packet, + local, target, 31000, 443, 0x12345678) + }) +} diff --git a/traceroute/icmp_unix.go b/traceroute/icmp_unix.go new file mode 100644 index 00000000..68c98ac3 --- /dev/null +++ b/traceroute/icmp_unix.go @@ -0,0 +1,180 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build !windows + +package traceroute + +import ( + "context" + "errors" + "net" + "sync/atomic" + "time" + + "golang.org/x/net/icmp" + "golang.org/x/net/ipv4" +) + +var nextICMPID = uint32(time.Now().UnixNano()) //nolint:gochecknoglobals,gosec + +func traceICMP(ctx context.Context, target net.IP, cfg options) (Result, error) { + conn, err := icmp.ListenPacket("ip4:icmp", "0.0.0.0") + if err != nil { + return Result{}, err + } + defer conn.Close() //nolint:errcheck + + result := Result{Routes: make([]*Route, 0, cfg.maxTTL)} + buffer := make([]byte, 2048) + for ttl := 1; ttl <= cfg.maxTTL; ttl++ { + replies := make([]probeReply, 0, cfg.attempts) + hopReached := false + for attempt := 0; attempt < cfg.attempts; attempt++ { + if err := ctx.Err(); err != nil { + return result, err + } + token := atomic.AddUint32(&nextICMPID, 1) + id := int(uint16(token >> 16)) + sequence := int(uint16(token)) + reply, err := sendICMPProbe(ctx, conn, buffer, target, ttl, id, sequence, cfg.timeout) + if err != nil { + if len(replies) > 0 { + result.Routes = append(result.Routes, repliesToRoute(replies)) + } + return result, err + } + replies = append(replies, reply) + hopReached = hopReached || reply.reached + } + result.Routes = append(result.Routes, repliesToRoute(replies)) + if hopReached { + result.Reached = true + return result, nil + } + } + return result, nil +} + +func sendICMPProbe(ctx context.Context, conn *icmp.PacketConn, buffer []byte, target net.IP, + ttl, id, sequence int, timeout time.Duration, +) (probeReply, error) { + if err := conn.IPv4PacketConn().SetTTL(ttl); err != nil { + return probeReply{}, err + } + message, err := (&icmp.Message{ + Type: ipv4.ICMPTypeEcho, + Body: &icmp.Echo{ID: id, Seq: sequence}, + }).Marshal(nil) + if err != nil { + return probeReply{}, err + } + startedAt := time.Now() + if _, err := conn.WriteTo(message, &net.IPAddr{IP: target}); err != nil { + return probeReply{}, err + } + deadline := probeDeadline(ctx, startedAt, timeout) + for { + if err := conn.SetReadDeadline(probeReadDeadline(ctx, deadline)); err != nil { + return probeReply{}, err + } + n, peer, err := conn.ReadFrom(buffer) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return probeReply{}, ctxErr + } + if errors.Is(err, net.ErrClosed) { + return probeReply{}, err + } + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + if time.Now().Before(deadline) { + continue + } + return probeReply{timedOut: true}, nil + } + return probeReply{}, err + } + from, ok := ipFromAddr(peer) + if !ok || !matchesICMPProbe(buffer[:n], from, target, id, sequence) { + continue + } + return probeReply{ + ip: from, + rtt: time.Since(startedAt), + reached: from.Equal(target), + }, nil + } +} + +func matchesICMPProbe(packet []byte, from, target net.IP, id, sequence int) bool { + message, err := icmp.ParseMessage(1, packet) + if err != nil { + return false + } + if message.Type == ipv4.ICMPTypeEchoReply { + echo, ok := message.Body.(*icmp.Echo) + return ok && from.Equal(target) && echo.ID == id && echo.Seq == sequence + } + quoted := quotedDatagram(message) + if len(quoted) < ipv4.HeaderLen || quoted[0]>>4 != ipv4.Version { + return false + } + header, err := ipv4.ParseHeader(quoted) + if err != nil || header.Protocol != 1 || len(quoted) < header.Len+8 || !header.Dst.Equal(target) { + return false + } + inner, err := icmp.ParseMessage(1, quoted[header.Len:]) + if err != nil { + return false + } + echo, ok := inner.Body.(*icmp.Echo) + return ok && echo.ID == id && echo.Seq == sequence +} + +func quotedDatagram(message *icmp.Message) []byte { + switch body := message.Body.(type) { + case *icmp.TimeExceeded: + return body.Data + case *icmp.DstUnreach: + return body.Data + case *icmp.ParamProb: + return body.Data + default: + return nil + } +} + +func ipFromAddr(addr net.Addr) (net.IP, bool) { + switch value := addr.(type) { + case *net.IPAddr: + if value != nil && value.IP != nil { + return value.IP, true + } + case *net.UDPAddr: + if value != nil && value.IP != nil { + return value.IP, true + } + } + return nil, false +} + +func probeDeadline(ctx context.Context, startedAt time.Time, timeout time.Duration) time.Time { + deadline := startedAt.Add(timeout) + if ctxDeadline, ok := ctx.Deadline(); ok && ctxDeadline.Before(deadline) { + return ctxDeadline + } + return deadline +} + +func probeReadDeadline(ctx context.Context, deadline time.Time) time.Time { + if ctx.Done() == nil { + return deadline + } + pollDeadline := time.Now().Add(100 * time.Millisecond) + if pollDeadline.Before(deadline) { + return pollDeadline + } + return deadline +} diff --git a/traceroute/integration_linux_test.go b/traceroute/integration_linux_test.go new file mode 100644 index 00000000..38868dea --- /dev/null +++ b/traceroute/integration_linux_test.go @@ -0,0 +1,175 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build linux + +package traceroute + +import ( + "context" + "errors" + "net" + "os" + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTCPTraceIntegration(t *testing.T) { + if os.Getenv("CLIUTILS_TRACEROUTE_INTEGRATION") == "" { + t.Skip("set CLIUTILS_TRACEROUTE_INTEGRATION=1 to run raw socket integration tests") + } + target := net.ParseIP("127.0.0.1").To4() + listener, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: target}) + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + port := listener.Addr().(*net.TCPAddr).Port + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + result, err := Trace(ctx, target, Options{ + Protocol: ProtocolTCP, + Port: uint16(port), + MaxTTL: 1, + Attempts: 1, + Timeout: time.Second, + }) + require.NoError(t, err) + require.Len(t, result.Routes, 1) + require.Len(t, result.Routes[0].Items, 1) + assert.True(t, result.Reached) + assert.Equal(t, target.String(), result.Routes[0].Items[0].IP) +} + +func TestTCPTraceRemoteHopIntegration(t *testing.T) { + address := os.Getenv("CLIUTILS_TRACEROUTE_REMOTE") + if address == "" { + t.Skip("set CLIUTILS_TRACEROUTE_REMOTE=IPv4:port to test a live intermediate hop") + } + host, portText, err := net.SplitHostPort(address) + require.NoError(t, err) + target := net.ParseIP(host).To4() + require.NotNil(t, target) + port, err := strconv.ParseUint(portText, 10, 16) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + result, err := Trace(ctx, target, Options{ + Protocol: ProtocolTCP, + Port: uint16(port), + MaxTTL: 2, + Attempts: 1, + Timeout: 2 * time.Second, + }) + require.NoError(t, err) + require.NotEmpty(t, result.Routes) + require.NotEmpty(t, result.Routes[0].Items) + assert.NotEqual(t, "*", result.Routes[0].Items[0].IP) +} + +func TestTCPTraceConcurrentIntegration(t *testing.T) { + if os.Getenv("CLIUTILS_TRACEROUTE_INTEGRATION") == "" { + t.Skip("set CLIUTILS_TRACEROUTE_INTEGRATION=1 to run raw socket integration tests") + } + + target := net.ParseIP("127.0.0.1").To4() + listener, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: target}) + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + port := uint16(listener.Addr().(*net.TCPAddr).Port) + + const workers = 64 + start := make(chan struct{}) + errorsCh := make(chan error, workers) + var waitGroup sync.WaitGroup + for range workers { + waitGroup.Add(1) + go func() { + defer waitGroup.Done() + <-start + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + result, err := Trace(ctx, target, Options{ + Protocol: ProtocolTCP, + Port: port, + MaxTTL: 1, + Attempts: 1, + Timeout: time.Second, + }) + if err != nil { + errorsCh <- err + return + } + if !result.Reached { + errorsCh <- errors.New("TCP traceroute did not reach local listener") + } + }() + } + close(start) + waitGroup.Wait() + close(errorsCh) + for err := range errorsCh { + t.Error(err) + } +} + +func TestUDPTraceLateApplicationResponseIntegration(t *testing.T) { + if os.Getenv("CLIUTILS_TRACEROUTE_INTEGRATION") == "" { + t.Skip("set CLIUTILS_TRACEROUTE_INTEGRATION=1 to run raw socket integration tests") + } + + target := net.ParseIP("127.0.0.1").To4() + server, err := net.ListenUDP("udp4", &net.UDPAddr{IP: target}) + require.NoError(t, err) + t.Cleanup(func() { _ = server.Close() }) + require.NoError(t, server.SetReadDeadline(time.Now().Add(2*time.Second))) + serverPort := uint16(server.LocalAddr().(*net.UDPAddr).Port) + + peers := make(chan *net.UDPAddr, 2) + serverErrors := make(chan error, 2) + go func() { + var buffer [MaxUDPProbePayload]byte + for probe := 0; probe < 2; probe++ { + _, peer, err := server.ReadFromUDP(buffer[:]) + if err != nil { + serverErrors <- err + return + } + peer = &net.UDPAddr{IP: append(net.IP(nil), peer.IP...), Port: peer.Port, Zone: peer.Zone} + peers <- peer + if probe == 0 { + go func() { + time.Sleep(220 * time.Millisecond) + _, err := server.WriteToUDP([]byte("late"), peer) + serverErrors <- err + }() + } + } + }() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + result, err := Trace(ctx, target, Options{ + Protocol: ProtocolUDP, + Port: serverPort, + MaxTTL: 1, + Attempts: 2, + Timeout: 150 * time.Millisecond, + }) + require.NoError(t, err) + require.Len(t, result.Routes, 1) + assert.False(t, result.Reached) + assert.Equal(t, 2, result.Routes[0].Failed) + + firstPeer := <-peers + secondPeer := <-peers + assert.NotEqual(t, firstPeer.Port, secondPeer.Port) + require.NoError(t, <-serverErrors) +} diff --git a/traceroute/protocol_linux_test.go b/traceroute/protocol_linux_test.go new file mode 100644 index 00000000..4841bf17 --- /dev/null +++ b/traceroute/protocol_linux_test.go @@ -0,0 +1,365 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build linux + +package traceroute + +import ( + "context" + "encoding/binary" + "net" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/bpf" + "golang.org/x/net/icmp" + "golang.org/x/net/ipv4" +) + +func TestProbeUDPValidation(t *testing.T) { + canceledContext, cancel := context.WithCancel(context.Background()) + cancel() + _, err := ProbeUDP(canceledContext, net.ParseIP("192.0.2.1"), 53, 1, time.Second) + require.ErrorIs(t, err, context.Canceled) + + _, err = ProbeUDP(context.Background(), net.ParseIP("2001:db8::1"), 53, 1, time.Second) + require.ErrorContains(t, err, "must be IPv4") + + for _, target := range []string{"0.0.0.0", "224.0.0.1", "255.255.255.255"} { + _, err = ProbeUDP(context.Background(), net.ParseIP(target), 53, 1, time.Second) + require.ErrorContains(t, err, "unicast") + } + + _, err = ProbeUDP(context.Background(), net.ParseIP("192.0.2.1"), 0, 1, time.Second) + require.ErrorContains(t, err, "destination port") + + _, err = ProbeUDP(context.Background(), net.ParseIP("192.0.2.1"), 53, MaxUDPHops+1, time.Second) + require.ErrorContains(t, err, "TTL is out of range") +} + +func TestClosedUDPProber(t *testing.T) { + prober := &UDPProber{} + require.NoError(t, prober.Close()) + _, err := prober.Probe(context.Background(), net.ParseIP("192.0.2.1"), 53, 1, time.Second) + require.ErrorContains(t, err, "closed") +} + +func TestParseUDPICMPReply(t *testing.T) { + target := net.ParseIP("203.0.113.10").To4() + quoted := quotedTransportPacket(t, 17, target, 31000, 33434, 9, 0) + packet := marshalICMP(t, ipv4.ICMPTypeTimeExceeded, &icmp.TimeExceeded{Data: quoted}) + + reply, matched := parseUDPICMPReply(packet, &net.IPAddr{IP: net.ParseIP("192.0.2.1")}, + target, 31000, 33434, 9, 2*time.Millisecond) + require.True(t, matched) + assert.Equal(t, "192.0.2.1", reply.ip.String()) + assert.Equal(t, 2*time.Millisecond, reply.rtt) + assert.False(t, reply.reached) + + _, matched = parseUDPICMPReply(packet, &net.IPAddr{IP: net.ParseIP("192.0.2.1")}, + target, 31001, 33434, 9, time.Millisecond) + assert.False(t, matched) + + _, matched = parseUDPICMPReply(packet, &net.IPAddr{IP: net.ParseIP("192.0.2.1")}, + target, 31000, 33434, 10, time.Millisecond) + assert.False(t, matched) + + unreachable := marshalICMP(t, ipv4.ICMPTypeDestinationUnreachable, &icmp.DstUnreach{Data: quoted}) + reply, matched = parseUDPICMPReply(unreachable, &net.IPAddr{IP: target}, + target, 31000, 33434, 9, time.Millisecond) + require.True(t, matched) + assert.True(t, reply.reached) +} + +func TestUDPPayloadUsesProbeIDAsQuotedLength(t *testing.T) { + payload, udpLength, err := udpPayload(7) + require.NoError(t, err) + assert.Len(t, payload, 7) + assert.Equal(t, uint16(15), udpLength) + + _, _, err = udpPayload(0) + require.Error(t, err) + _, _, err = udpPayload(MaxUDPProbePayload + 1) + require.ErrorContains(t, err, "invalid") +} + +func TestUDPLeaseRegistryQuarantine(t *testing.T) { + now := time.Unix(100, 0) + registry := udpLeaseRegistry{leases: make(map[udpLeaseKey]time.Time)} + key := udpLeaseKey{ + target: [net.IPv4len]byte{127, 0, 0, 1}, + targetPort: 33434, + sourcePort: 31000, + } + + require.True(t, registry.reserve(key, now)) + assert.False(t, registry.reserve(key, now)) + registry.finish(key, true, now) + assert.False(t, registry.reserve(key, now.Add(MaxTimeout-time.Nanosecond))) + + differentEndpoint := key + differentEndpoint.targetPort++ + require.True(t, registry.reserve(differentEndpoint, now)) + registry.finish(differentEndpoint, false, now) + assert.True(t, registry.reserve(key, now.Add(MaxTimeout))) +} + +func TestUDPLeaseRegistryConcurrentReservation(t *testing.T) { + registry := udpLeaseRegistry{leases: make(map[udpLeaseKey]time.Time)} + key := udpLeaseKey{ + target: [net.IPv4len]byte{127, 0, 0, 1}, + targetPort: 33434, + sourcePort: 31000, + } + start := make(chan struct{}) + var successes atomic.Int32 + var waitGroup sync.WaitGroup + for range 32 { + waitGroup.Add(1) + go func() { + defer waitGroup.Done() + <-start + if registry.reserve(key, time.Unix(100, 0)) { + successes.Add(1) + } + }() + } + close(start) + waitGroup.Wait() + assert.Equal(t, int32(1), successes.Load()) +} + +func TestUDPLeaseRegistryDefersBulkCleanup(t *testing.T) { + now := time.Unix(100, 0) + registry := udpLeaseRegistry{leases: make(map[udpLeaseKey]time.Time)} + for port := 1; port <= 10000; port++ { + registry.leases[udpLeaseKey{sourcePort: uint16(port)}] = now.Add(-time.Second) + } + newKey := udpLeaseKey{sourcePort: 31000} + require.True(t, registry.reserve(newKey, now)) + assert.Len(t, registry.leases, 10001, "reserve must not scan all unrelated leases") + + registry.cleanupExpiredAt(now) + assert.Equal(t, map[udpLeaseKey]time.Time{newKey: {}}, registry.leases) +} + +func TestUDPProbeSocketsUseDistinctPorts(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + target := net.ParseIP("127.0.0.1").To4() + ports := make(map[uint16]struct{}) + + for range 64 { + socket, err := openUDPProbeSocket(ctx, target, 33434) + require.NoError(t, err) + _, duplicate := ports[socket.key.sourcePort] + assert.False(t, duplicate) + ports[socket.key.sourcePort] = struct{}{} + socket.sent = true + socket.close() + } + assert.Len(t, ports, 64) +} + +func TestLateUDPApplicationReplyDoesNotReachNextProbe(t *testing.T) { + server, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + require.NoError(t, err) + t.Cleanup(func() { _ = server.Close() }) + serverAddr := server.LocalAddr().(*net.UDPAddr) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + first, err := openUDPProbeSocket(ctx, serverAddr.IP, uint16(serverAddr.Port)) + require.NoError(t, err) + firstPort := first.key.sourcePort + first.sent = true + first.close() + + second, err := openUDPProbeSocket(ctx, serverAddr.IP, uint16(serverAddr.Port)) + require.NoError(t, err) + t.Cleanup(second.close) + require.NotEqual(t, firstPort, second.key.sourcePort) + + writeErr := make(chan error, 1) + go func() { + time.Sleep(10 * time.Millisecond) + _, err := server.WriteToUDP([]byte("late"), &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(firstPort)}) + writeErr <- err + }() + reply, err := waitUDPApplicationReply(ctx, second.conn, serverAddr.IP, + uint16(serverAddr.Port), time.Now(), 50*time.Millisecond) + require.NoError(t, err) + require.NoError(t, <-writeErr) + assert.True(t, reply.timedOut) +} + +func TestParseTCPICMPReply(t *testing.T) { + target := net.ParseIP("203.0.113.10").To4() + const sequence = 0x12345678 + quoted := quotedTransportPacket(t, 6, target, 31000, 443, 0, sequence) + packet := marshalICMP(t, ipv4.ICMPTypeTimeExceeded, &icmp.TimeExceeded{Data: quoted}) + + reply, matched := parseTCPICMPReply(packet, &net.IPAddr{IP: net.ParseIP("192.0.2.1")}, + target, 31000, 443, sequence, 2*time.Millisecond) + require.True(t, matched) + assert.Equal(t, "192.0.2.1", reply.ip.String()) + assert.False(t, reply.reached) + + _, matched = parseTCPICMPReply(packet, &net.IPAddr{IP: net.ParseIP("192.0.2.1")}, + target, 31000, 443, sequence+1, time.Millisecond) + assert.False(t, matched) +} + +func TestMatchesICMPProbeUsesFullTokenAndTarget(t *testing.T) { + target := net.ParseIP("203.0.113.10").To4() + peer := net.ParseIP("192.0.2.1").To4() + const id, sequence = 17, 23 + echoReply := marshalICMP(t, ipv4.ICMPTypeEchoReply, &icmp.Echo{ID: id, Seq: sequence}) + assert.True(t, matchesICMPProbe(echoReply, target, target, id, sequence)) + assert.False(t, matchesICMPProbe(echoReply, peer, target, id, sequence)) + assert.False(t, matchesICMPProbe(echoReply, target, target, id, sequence+1)) + + inner, err := (&icmp.Message{ + Type: ipv4.ICMPTypeEcho, + Body: &icmp.Echo{ID: id, Seq: sequence}, + }).Marshal(nil) + require.NoError(t, err) + header := &ipv4.Header{ + Version: ipv4.Version, + Len: ipv4.HeaderLen, + TotalLen: ipv4.HeaderLen + len(inner), + Protocol: 1, + Src: net.ParseIP("192.0.2.20").To4(), + Dst: target, + } + encodedHeader, err := header.Marshal() + require.NoError(t, err) + timeExceeded := marshalICMP(t, ipv4.ICMPTypeTimeExceeded, + &icmp.TimeExceeded{Data: append(encodedHeader, inner...)}) + assert.True(t, matchesICMPProbe(timeExceeded, peer, target, id, sequence)) + assert.False(t, matchesICMPProbe(timeExceeded, peer, + net.ParseIP("203.0.113.11").To4(), id, sequence)) +} + +func TestTCPPacketEncodingAndResponseMatching(t *testing.T) { + source := net.ParseIP("192.0.2.20").To4() + target := net.ParseIP("203.0.113.10").To4() + const sequence = 0x12345678 + segment := marshalTCPSYN(source, target, 31000, 443, sequence) + + assert.Equal(t, uint16(31000), binary.BigEndian.Uint16(segment[0:2])) + assert.Equal(t, uint16(443), binary.BigEndian.Uint16(segment[2:4])) + assert.Equal(t, uint32(sequence), binary.BigEndian.Uint32(segment[4:8])) + assert.Equal(t, byte(tcpFlagSYN), segment[13]) + assert.Equal(t, uint16(0), tcpChecksum(source, target, segment)) + + response := make([]byte, 20) + binary.BigEndian.PutUint16(response[0:2], 443) + binary.BigEndian.PutUint16(response[2:4], 31000) + binary.BigEndian.PutUint32(response[8:12], sequence+1) + response[12] = 5 << 4 + response[13] = tcpFlagSYN | tcpFlagACK + header := &ipv4.Header{Src: target, Dst: source} + assert.True(t, matchesTCPResponse(header, response, source, target, 31000, 443, sequence)) + + response[13] = tcpFlagACK + assert.False(t, matchesTCPResponse(header, response, source, target, 31000, 443, sequence)) + + response[13] = tcpFlagRST | tcpFlagACK + assert.True(t, matchesTCPResponse(header, response, source, target, 31000, 443, sequence)) + + response[8]++ + assert.False(t, matchesTCPResponse(header, response, source, target, 31000, 443, sequence)) +} + +func TestTCPPacketFilter(t *testing.T) { + local := net.ParseIP("192.0.2.20").To4() + target := net.ParseIP("203.0.113.10").To4() + instructions, err := tcpPacketFilter(local, target, 31000, 443) + require.NoError(t, err) + vm, err := bpf.NewVM(instructions) + require.NoError(t, err) + + packet := tcpResponsePacket(t, target, local, 443, 31000, tcpFlagSYN|tcpFlagACK) + accepted, err := vm.Run(packet) + require.NoError(t, err) + assert.Equal(t, tcpFilterSnapshotLength, accepted) + + packet = tcpResponsePacket(t, net.ParseIP("203.0.113.11").To4(), local, + 443, 31000, tcpFlagSYN|tcpFlagACK) + accepted, err = vm.Run(packet) + require.NoError(t, err) + assert.Zero(t, accepted) + + packet = tcpResponsePacket(t, target, local, 443, 31000, tcpFlagACK) + accepted, err = vm.Run(packet) + require.NoError(t, err) + assert.Zero(t, accepted) + + packet = tcpResponsePacket(t, target, local, 443, 31000, tcpFlagSYN|tcpFlagACK) + packet[7] = 1 + accepted, err = vm.Run(packet) + require.NoError(t, err) + assert.Zero(t, accepted) +} + +func tcpResponsePacket(t *testing.T, source, target net.IP, sourcePort, targetPort uint16, + flags byte, +) []byte { + t.Helper() + header := &ipv4.Header{ + Version: ipv4.Version, + Len: ipv4.HeaderLen, + TotalLen: ipv4.HeaderLen + 20, + Protocol: 6, + Src: source, + Dst: target, + } + encodedHeader, err := header.Marshal() + require.NoError(t, err) + tcpHeader := make([]byte, 20) + binary.BigEndian.PutUint16(tcpHeader[0:2], sourcePort) + binary.BigEndian.PutUint16(tcpHeader[2:4], targetPort) + tcpHeader[12] = 5 << 4 + tcpHeader[13] = flags + return append(encodedHeader, tcpHeader...) +} + +func quotedTransportPacket(t *testing.T, protocol int, target net.IP, + sourcePort, targetPort, udpLength uint16, sequence uint32, +) []byte { + t.Helper() + header := &ipv4.Header{ + Version: ipv4.Version, + Len: ipv4.HeaderLen, + TotalLen: ipv4.HeaderLen + 8, + Protocol: protocol, + Src: net.ParseIP("192.0.2.20").To4(), + Dst: target, + } + encoded, err := header.Marshal() + require.NoError(t, err) + transport := make([]byte, 8) + binary.BigEndian.PutUint16(transport[0:2], sourcePort) + binary.BigEndian.PutUint16(transport[2:4], targetPort) + if protocol == 17 { + binary.BigEndian.PutUint16(transport[4:6], udpLength) + } else { + binary.BigEndian.PutUint32(transport[4:8], sequence) + } + return append(encoded, transport...) +} + +func marshalICMP(t *testing.T, messageType icmp.Type, body icmp.MessageBody) []byte { + t.Helper() + packet, err := (&icmp.Message{Type: messageType, Body: body}).Marshal(nil) + require.NoError(t, err) + return packet +} diff --git a/traceroute/tcp_filter_linux.go b/traceroute/tcp_filter_linux.go new file mode 100644 index 00000000..e875cb03 --- /dev/null +++ b/traceroute/tcp_filter_linux.go @@ -0,0 +1,96 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build linux + +package traceroute + +import ( + "encoding/binary" + "errors" + "fmt" + "net" + + "golang.org/x/net/bpf" + "golang.org/x/sys/unix" +) + +const tcpFilterSnapshotLength = 128 + +func setTCPPacketFilter(conn *net.IPConn, localIP, targetIP net.IP, + sourcePort, targetPort uint16, +) error { + instructions, err := tcpPacketFilter(localIP, targetIP, sourcePort, targetPort) + if err != nil { + return err + } + rawInstructions, err := bpf.Assemble(instructions) + if err != nil { + return fmt.Errorf("assemble TCP traceroute packet filter: %w", err) + } + filters := make([]unix.SockFilter, len(rawInstructions)) + for index, instruction := range rawInstructions { + filters[index] = unix.SockFilter{ + Code: instruction.Op, + Jt: instruction.Jt, + Jf: instruction.Jf, + K: instruction.K, + } + } + program := unix.SockFprog{Len: uint16(len(filters)), Filter: &filters[0]} + rawConn, err := conn.SyscallConn() + if err != nil { + return fmt.Errorf("get TCP traceroute raw socket: %w", err) + } + var attachErr error + if err := rawConn.Control(func(fd uintptr) { + attachErr = unix.SetsockoptSockFprog(int(fd), unix.SOL_SOCKET, unix.SO_ATTACH_FILTER, &program) + }); err != nil { + return fmt.Errorf("control TCP traceroute raw socket: %w", err) + } + if attachErr != nil { + return fmt.Errorf("attach TCP traceroute packet filter: %w", attachErr) + } + return nil +} + +func tcpPacketFilter(localIP, targetIP net.IP, sourcePort, targetPort uint16) ([]bpf.Instruction, error) { + local := localIP.To4() + target := targetIP.To4() + if local == nil || target == nil { + return nil, errors.New("TCP traceroute packet filter requires IPv4 addresses") + } + checks := []bpf.Instruction{ + bpf.LoadAbsolute{Off: 9, Size: 1}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: 6}, + bpf.LoadAbsolute{Off: 12, Size: 4}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: binary.BigEndian.Uint32(target)}, + bpf.LoadAbsolute{Off: 16, Size: 4}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: binary.BigEndian.Uint32(local)}, + bpf.LoadAbsolute{Off: 6, Size: 2}, + bpf.JumpIf{Cond: bpf.JumpBitsNotSet, Val: 0x1fff}, + bpf.LoadMemShift{Off: 0}, + bpf.LoadIndirect{Off: 0, Size: 2}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: uint32(targetPort)}, + bpf.LoadIndirect{Off: 2, Size: 2}, + bpf.JumpIf{Cond: bpf.JumpEqual, Val: uint32(sourcePort)}, + bpf.LoadIndirect{Off: 13, Size: 1}, + bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: tcpFlagACK}, + bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: tcpFlagSYN | tcpFlagRST}, + } + dropIndex := len(checks) + 1 + for index, instruction := range checks { + jump, ok := instruction.(bpf.JumpIf) + if !ok { + continue + } + jump.SkipFalse = uint8(dropIndex - index - 1) + checks[index] = jump + } + return append(checks, + bpf.RetConstant{Val: tcpFilterSnapshotLength}, + bpf.RetConstant{Val: 0}, + ), nil +} diff --git a/traceroute/tcp_other.go b/traceroute/tcp_other.go new file mode 100644 index 00000000..e9394da2 --- /dev/null +++ b/traceroute/tcp_other.go @@ -0,0 +1,18 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build !linux && !windows + +package traceroute + +import ( + "context" + "errors" + "net" +) + +func traceTCP(context.Context, net.IP, options) (Result, error) { + return Result{}, errors.New("TCP traceroute is only supported on linux") +} diff --git a/traceroute/tcp_unix.go b/traceroute/tcp_unix.go new file mode 100644 index 00000000..ee80e0c3 --- /dev/null +++ b/traceroute/tcp_unix.go @@ -0,0 +1,366 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build linux + +package traceroute + +import ( + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "net" + "time" + + "golang.org/x/net/icmp" + "golang.org/x/net/ipv4" +) + +const ( + tcpFlagRST = 0x04 + tcpFlagSYN = 0x02 + tcpFlagACK = 0x10 +) + +type tcpProbeSocket struct { + localIP net.IP + sourcePort uint16 + listener *net.TCPListener + conn *net.IPConn + raw *ipv4.RawConn + icmp *icmp.PacketConn + tcpBuffer [2048]byte + icmpBuffer [2048]byte +} + +type tcpWaitResult struct { + reply probeReply + err error +} + +func traceTCP(ctx context.Context, target net.IP, cfg options) (Result, error) { + socket, err := openTCPProbeSocket(ctx, target, cfg.port) + if err != nil { + return Result{}, err + } + defer socket.close() + + result := Result{Routes: make([]*Route, 0, cfg.maxTTL)} + for ttl := 1; ttl <= cfg.maxTTL; ttl++ { + replies := make([]probeReply, 0, cfg.attempts) + hopReached := false + for attempt := 0; attempt < cfg.attempts; attempt++ { + reply, err := sendTCPProbe(ctx, socket, target, cfg.port, ttl, cfg.timeout) + if err != nil { + if len(replies) > 0 { + result.Routes = append(result.Routes, repliesToRoute(replies)) + } + return result, err + } + replies = append(replies, reply) + hopReached = hopReached || reply.reached + } + result.Routes = append(result.Routes, repliesToRoute(replies)) + if hopReached { + result.Reached = true + return result, nil + } + } + return result, nil +} + +func openTCPProbeSocket(ctx context.Context, target net.IP, targetPort uint16) (*tcpProbeSocket, error) { + localIP, err := routeSourceIPv4(ctx, target, targetPort) + if err != nil { + return nil, err + } + listener, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: localIP}) + if err != nil { + return nil, err + } + localAddr, ok := listener.Addr().(*net.TCPAddr) + if !ok || localAddr.Port <= 0 || localAddr.Port > 65535 { + _ = listener.Close() + return nil, errors.New("get TCP traceroute source port") + } + conn, err := net.ListenIP("ip4:tcp", &net.IPAddr{IP: localIP}) + if err != nil { + _ = listener.Close() + return nil, err + } + raw, err := ipv4.NewRawConn(conn) + if err != nil { + _ = conn.Close() + _ = listener.Close() + return nil, err + } + if err := setTCPPacketFilter(conn, localIP, target, uint16(localAddr.Port), targetPort); err != nil { + _ = conn.Close() + _ = listener.Close() + return nil, err + } + icmpConn, err := icmp.ListenPacket("ip4:icmp", "0.0.0.0") + if err != nil { + _ = conn.Close() + _ = listener.Close() + return nil, err + } + return &tcpProbeSocket{ + localIP: localIP, + sourcePort: uint16(localAddr.Port), + listener: listener, + conn: conn, + raw: raw, + icmp: icmpConn, + }, nil +} + +func routeSourceIPv4(ctx context.Context, target net.IP, port uint16) (net.IP, error) { + dialer := net.Dialer{} + conn, err := dialer.DialContext(ctx, "udp4", net.JoinHostPort(target.String(), integerString(port))) + if err != nil { + return nil, err + } + defer conn.Close() //nolint:errcheck + addr, ok := conn.LocalAddr().(*net.UDPAddr) + if !ok || addr.IP.To4() == nil { + return nil, errors.New("determine TCP traceroute source IPv4 address") + } + return addr.IP.To4(), nil +} + +func (socket *tcpProbeSocket) close() { + _ = socket.icmp.Close() + _ = socket.conn.Close() + _ = socket.listener.Close() +} + +func sendTCPProbe(ctx context.Context, socket *tcpProbeSocket, target net.IP, + targetPort uint16, ttl int, timeout time.Duration, +) (probeReply, error) { + if err := ctx.Err(); err != nil { + return probeReply{}, err + } + var sequenceBytes [4]byte + if _, err := rand.Read(sequenceBytes[:]); err != nil { + return probeReply{}, fmt.Errorf("generate TCP traceroute sequence: %w", err) + } + sequence := binary.BigEndian.Uint32(sequenceBytes[:]) + segment := marshalTCPSYN(socket.localIP, target, socket.sourcePort, targetPort, sequence) + header := &ipv4.Header{ + Version: ipv4.Version, + Len: ipv4.HeaderLen, + TotalLen: ipv4.HeaderLen + len(segment), + ID: int(sequence & 0xffff), + TTL: ttl, + Protocol: 6, + Src: socket.localIP, + Dst: target, + } + startedAt := time.Now() + if err := socket.raw.WriteTo(header, segment, nil); err != nil { + return probeReply{}, err + } + return waitTCPReply(ctx, socket, target, targetPort, sequence, startedAt, timeout) +} + +func marshalTCPSYN(sourceIP, targetIP net.IP, sourcePort, targetPort uint16, sequence uint32) []byte { + segment := make([]byte, 20) + binary.BigEndian.PutUint16(segment[0:2], sourcePort) + binary.BigEndian.PutUint16(segment[2:4], targetPort) + binary.BigEndian.PutUint32(segment[4:8], sequence) + segment[12] = 5 << 4 + segment[13] = tcpFlagSYN + binary.BigEndian.PutUint16(segment[14:16], 64240) + binary.BigEndian.PutUint16(segment[16:18], tcpChecksum(sourceIP, targetIP, segment)) + return segment +} + +func tcpChecksum(sourceIP, targetIP net.IP, segment []byte) uint16 { + pseudo := make([]byte, 12+len(segment)) + copy(pseudo[0:4], sourceIP.To4()) + copy(pseudo[4:8], targetIP.To4()) + pseudo[9] = 6 + binary.BigEndian.PutUint16(pseudo[10:12], uint16(len(segment))) + copy(pseudo[12:], segment) + var sum uint32 + for index := 0; index+1 < len(pseudo); index += 2 { + sum += uint32(binary.BigEndian.Uint16(pseudo[index : index+2])) + } + if len(pseudo)%2 != 0 { + sum += uint32(pseudo[len(pseudo)-1]) << 8 + } + for sum>>16 != 0 { + sum = sum&0xffff + sum>>16 + } + return ^uint16(sum) +} + +func waitTCPReply(ctx context.Context, socket *tcpProbeSocket, target net.IP, + targetPort uint16, sequence uint32, startedAt time.Time, timeout time.Duration, +) (probeReply, error) { + waitCtx, cancel := context.WithCancel(ctx) + results := make(chan tcpWaitResult, 2) + go func() { + reply, err := waitTCPPacket(waitCtx, socket, target, targetPort, sequence, startedAt, timeout) + results <- tcpWaitResult{reply: reply, err: err} + }() + go func() { + reply, err := waitTCPICMP(waitCtx, socket, target, targetPort, sequence, startedAt, timeout) + results <- tcpWaitResult{reply: reply, err: err} + }() + + var firstErr error + for pending := 2; pending > 0; pending-- { + result := <-results + if result.err == nil && !result.reply.timedOut { + cancel() + _ = socket.conn.SetReadDeadline(time.Now()) + _ = socket.icmp.SetReadDeadline(time.Now()) + for pending--; pending > 0; pending-- { + <-results + } + return result.reply, nil + } + if result.err != nil && firstErr == nil { + firstErr = result.err + } + } + cancel() + if err := ctx.Err(); err != nil { + return probeReply{}, err + } + if firstErr != nil { + return probeReply{}, firstErr + } + return probeReply{timedOut: true}, nil +} + +func waitTCPPacket(ctx context.Context, socket *tcpProbeSocket, target net.IP, + targetPort uint16, sequence uint32, startedAt time.Time, timeout time.Duration, +) (probeReply, error) { + deadline := probeDeadline(ctx, startedAt, timeout) + buf := socket.tcpBuffer[:] + for { + if err := socket.conn.SetReadDeadline(probeReadDeadline(ctx, deadline)); err != nil { + return probeReply{}, err + } + header, payload, _, err := socket.raw.ReadFrom(buf) + if err != nil { + if ctx.Err() != nil { + return probeReply{}, ctx.Err() + } + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + if time.Now().Before(deadline) { + continue + } + return probeReply{timedOut: true}, nil + } + return probeReply{}, err + } + if !matchesTCPResponse(header, payload, socket.localIP, target, socket.sourcePort, + targetPort, sequence) { + continue + } + return probeReply{ip: target, rtt: time.Since(startedAt), reached: true}, nil + } +} + +func matchesTCPResponse(header *ipv4.Header, segment []byte, localIP, target net.IP, + sourcePort, targetPort uint16, sequence uint32, +) bool { + if header == nil || !header.Src.Equal(target) || !header.Dst.Equal(localIP) || len(segment) < 20 { + return false + } + if binary.BigEndian.Uint16(segment[0:2]) != targetPort || + binary.BigEndian.Uint16(segment[2:4]) != sourcePort { + return false + } + flags := segment[13] + if flags&(tcpFlagSYN|tcpFlagRST) == 0 || flags&tcpFlagACK == 0 { + return false + } + return binary.BigEndian.Uint32(segment[8:12]) == sequence+1 +} + +func waitTCPICMP(ctx context.Context, socket *tcpProbeSocket, target net.IP, + targetPort uint16, sequence uint32, startedAt time.Time, timeout time.Duration, +) (probeReply, error) { + deadline := probeDeadline(ctx, startedAt, timeout) + buf := socket.icmpBuffer[:] + for { + if err := socket.icmp.SetReadDeadline(probeReadDeadline(ctx, deadline)); err != nil { + return probeReply{}, err + } + n, peer, err := socket.icmp.ReadFrom(buf) + if err != nil { + if ctx.Err() != nil { + return probeReply{}, ctx.Err() + } + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + if time.Now().Before(deadline) { + continue + } + return probeReply{timedOut: true}, nil + } + return probeReply{}, err + } + reply, matched := parseTCPICMPReply(buf[:n], peer, target, socket.sourcePort, + targetPort, sequence, time.Since(startedAt)) + if matched { + return reply, nil + } + } +} + +func parseTCPICMPReply(packet []byte, peer net.Addr, target net.IP, + sourcePort, targetPort uint16, sequence uint32, rtt time.Duration, +) (probeReply, bool) { + message, err := icmp.ParseMessage(1, packet) + if err != nil { + return probeReply{}, false + } + quoted := quotedDatagram(message) + if quoted == nil { + return probeReply{}, false + } + header, err := ipv4.ParseHeader(quoted) + if err != nil || header.Protocol != 6 || len(quoted) < header.Len+8 || !header.Dst.Equal(target) { + return probeReply{}, false + } + tcpHeader := quoted[header.Len : header.Len+8] + if binary.BigEndian.Uint16(tcpHeader[0:2]) != sourcePort || + binary.BigEndian.Uint16(tcpHeader[2:4]) != targetPort || + binary.BigEndian.Uint32(tcpHeader[4:8]) != sequence { + return probeReply{}, false + } + from, ok := ipFromAddr(peer) + if !ok { + return probeReply{}, false + } + _, destinationUnreachable := message.Body.(*icmp.DstUnreach) + return probeReply{ + ip: from, + rtt: rtt, + reached: destinationUnreachable && from.Equal(target), + }, true +} + +func integerString(value uint16) string { + const digits = "0123456789" + if value == 0 { + return "0" + } + var buf [5]byte + index := len(buf) + for value > 0 { + index-- + buf[index] = digits[value%10] + value /= 10 + } + return string(buf[index:]) +} diff --git a/traceroute/trace_unix.go b/traceroute/trace_unix.go new file mode 100644 index 00000000..6e125dc2 --- /dev/null +++ b/traceroute/trace_unix.go @@ -0,0 +1,26 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build !windows + +package traceroute + +import ( + "context" + "net" +) + +func trace(ctx context.Context, target net.IP, cfg options) (Result, error) { + switch cfg.protocol { + case ProtocolICMP: + return traceICMP(ctx, target, cfg) + case ProtocolUDP: + return traceUDP(ctx, target, cfg) + case ProtocolTCP: + return traceTCP(ctx, target, cfg) + default: + return Result{}, nil + } +} diff --git a/traceroute/trace_windows.go b/traceroute/trace_windows.go new file mode 100644 index 00000000..145d2a59 --- /dev/null +++ b/traceroute/trace_windows.go @@ -0,0 +1,40 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build windows + +package traceroute + +import ( + "context" + "errors" + "net" + "time" +) + +// UDPProber is unavailable on Windows. +type UDPProber struct{} + +func trace(context.Context, net.IP, options) (Result, error) { + return Result{}, errors.New("traceroute is not supported on windows") +} + +// ProbeUDP is unavailable on Windows. +func ProbeUDP(context.Context, net.IP, uint16, int, time.Duration) (ProbeResult, error) { + return ProbeResult{}, errors.New("UDP probe is not supported on windows") +} + +// NewUDPProber is unavailable on Windows. +func NewUDPProber() (*UDPProber, error) { + return nil, errors.New("UDP probe is not supported on windows") +} + +// Probe is unavailable on Windows. +func (*UDPProber) Probe(context.Context, net.IP, uint16, int, time.Duration) (ProbeResult, error) { + return ProbeResult{}, errors.New("UDP probe is not supported on windows") +} + +// Close is a no-op on Windows. +func (*UDPProber) Close() error { return nil } diff --git a/traceroute/traceroute.go b/traceroute/traceroute.go new file mode 100644 index 00000000..0097ed40 --- /dev/null +++ b/traceroute/traceroute.go @@ -0,0 +1,207 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +// Package traceroute discovers IPv4 network paths with ICMP, UDP, or TCP SYN probes. +package traceroute + +import ( + "context" + "errors" + "fmt" + "math" + "net" + "time" +) + +const ( + // MaxHops is the maximum hop count accepted for ICMP and TCP traces. + MaxHops = 60 + // MaxUDPHops is the maximum hop count accepted for UDP traces. + MaxUDPHops = 255 + // MaxAttempts is the maximum number of probes sent for each hop. + MaxAttempts = 10 + // MaxUDPProbePayload keeps IPv4 UDP probes below common path MTUs. + MaxUDPProbePayload = 1200 + // MaxTimeout is the maximum wait time for one probe. + MaxTimeout = 30 * time.Second +) + +const defaultTimeout = 500 * time.Millisecond + +// Protocol identifies the packet type used to discover a path. +type Protocol string + +const ( + ProtocolICMP Protocol = "icmp" + ProtocolUDP Protocol = "udp" + ProtocolTCP Protocol = "tcp" +) + +// Options controls a single traceroute run. +type Options struct { + Protocol Protocol + Port uint16 + MaxTTL int + Attempts int + Timeout time.Duration +} + +// RouteItem is one probe response at a hop. A timed-out probe uses IP "*". +type RouteItem struct { + IP string `json:"ip"` + ResponseTime float64 `json:"response_time"` +} + +// Route summarizes all probe responses at one hop. Durations are microseconds. +type Route struct { + Total int `json:"total"` + Failed int `json:"failed"` + Loss float64 `json:"loss"` + AvgCost float64 `json:"avg_cost"` + MinCost float64 `json:"min_cost"` + MaxCost float64 `json:"max_cost"` + StdCost float64 `json:"std_cost"` + Items []*RouteItem `json:"items"` +} + +// Result contains one route summary per TTL and whether the destination replied. +type Result struct { + Routes []*Route + Reached bool +} + +// ProbeResult describes one protocol probe. +type ProbeResult struct { + IP string + RTT time.Duration + Reached bool + TimedOut bool + Sent bool +} + +type probeReply struct { + ip net.IP + rtt time.Duration + reached bool + timedOut bool + sent bool +} + +type options struct { + protocol Protocol + port uint16 + maxTTL int + attempts int + timeout time.Duration +} + +// Trace discovers the path to an IPv4 destination. +func Trace(ctx context.Context, target net.IP, opts Options) (Result, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return Result{}, err + } + target = target.To4() + if target == nil { + return Result{}, errors.New("traceroute target must be IPv4") + } + if target.IsUnspecified() || target.IsMulticast() || target.Equal(net.IPv4bcast) { + return Result{}, fmt.Errorf("traceroute target %s must be a unicast address", target.String()) + } + cfg, err := normalizeOptions(opts) + if err != nil { + return Result{}, err + } + return trace(ctx, target, cfg) +} + +func normalizeOptions(opts Options) (options, error) { + cfg := options{ + protocol: opts.Protocol, + port: opts.Port, + maxTTL: opts.MaxTTL, + attempts: opts.Attempts, + timeout: opts.Timeout, + } + if cfg.protocol == "" { + cfg.protocol = ProtocolICMP + } + if cfg.protocol != ProtocolICMP && cfg.protocol != ProtocolUDP && cfg.protocol != ProtocolTCP { + return options{}, fmt.Errorf("unsupported traceroute protocol %q", cfg.protocol) + } + if (cfg.protocol == ProtocolUDP || cfg.protocol == ProtocolTCP) && cfg.port == 0 { + return options{}, fmt.Errorf("%s traceroute requires a destination port", cfg.protocol) + } + if cfg.maxTTL <= 0 { + cfg.maxTTL = 30 + } + maxTTL := MaxHops + if cfg.protocol == ProtocolUDP { + maxTTL = MaxUDPHops + } + if cfg.maxTTL > maxTTL { + cfg.maxTTL = maxTTL + } + if cfg.attempts <= 0 { + cfg.attempts = 1 + } + if cfg.attempts > MaxAttempts { + cfg.attempts = MaxAttempts + } + if cfg.protocol == ProtocolUDP && cfg.maxTTL*cfg.attempts > MaxUDPProbePayload { + return options{}, fmt.Errorf("UDP traceroute max TTL times attempts must not exceed %d", + MaxUDPProbePayload) + } + if cfg.timeout <= 0 { + cfg.timeout = defaultTimeout + } + if cfg.timeout > MaxTimeout { + cfg.timeout = MaxTimeout + } + return cfg, nil +} + +func repliesToRoute(replies []probeReply) *Route { + route := &Route{Total: len(replies), Items: make([]*RouteItem, 0, len(replies))} + latencies := make([]float64, 0, len(replies)) + for _, reply := range replies { + item := &RouteItem{IP: "*"} + if reply.timedOut || reply.ip == nil { + route.Failed++ + } else { + item.IP = reply.ip.String() + item.ResponseTime = float64(reply.rtt.Microseconds()) + latencies = append(latencies, item.ResponseTime) + } + route.Items = append(route.Items, item) + } + if route.Total > 0 { + route.Loss = float64(route.Failed) * 100 / float64(route.Total) + } + if len(latencies) == 0 { + return route + } + route.MinCost, route.MaxCost = latencies[0], latencies[0] + for _, latency := range latencies { + route.AvgCost += latency + if latency < route.MinCost { + route.MinCost = latency + } + if latency > route.MaxCost { + route.MaxCost = latency + } + } + route.AvgCost /= float64(len(latencies)) + if len(latencies) > 1 { + for _, latency := range latencies { + delta := latency - route.AvgCost + route.StdCost += delta * delta + } + route.StdCost = math.Sqrt(route.StdCost / float64(len(latencies)-1)) + } + return route +} diff --git a/traceroute/traceroute_test.go b/traceroute/traceroute_test.go new file mode 100644 index 00000000..3d9be6d4 --- /dev/null +++ b/traceroute/traceroute_test.go @@ -0,0 +1,80 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package traceroute + +import ( + "context" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTraceRejectsNonUnicastTargets(t *testing.T) { + for _, target := range []string{"0.0.0.0", "224.0.0.1", "255.255.255.255"} { + _, err := Trace(context.Background(), net.ParseIP(target), Options{}) + require.ErrorContains(t, err, "unicast") + } +} + +func TestNormalizeOptions(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + cfg, err := normalizeOptions(Options{}) + require.NoError(t, err) + assert.Equal(t, ProtocolICMP, cfg.protocol) + assert.Equal(t, 30, cfg.maxTTL) + assert.Equal(t, 1, cfg.attempts) + assert.Equal(t, defaultTimeout, cfg.timeout) + }) + + t.Run("protocol limits", func(t *testing.T) { + cfg, err := normalizeOptions(Options{ + Protocol: ProtocolUDP, + Port: 53, + MaxTTL: 1000, + Attempts: 4, + Timeout: time.Minute, + }) + require.NoError(t, err) + assert.Equal(t, MaxUDPHops, cfg.maxTTL) + assert.Equal(t, 4, cfg.attempts) + assert.Equal(t, MaxTimeout, cfg.timeout) + }) + + t.Run("UDP fragmentation guard", func(t *testing.T) { + _, err := normalizeOptions(Options{ + Protocol: ProtocolUDP, + Port: 53, + MaxTTL: MaxUDPHops, + Attempts: MaxAttempts, + }) + require.ErrorContains(t, err, "max TTL times attempts") + }) + + for _, protocol := range []Protocol{ProtocolTCP, ProtocolUDP} { + _, err := normalizeOptions(Options{Protocol: protocol}) + require.ErrorContains(t, err, "requires a destination port") + } +} + +func TestRepliesToRoute(t *testing.T) { + route := repliesToRoute([]probeReply{ + {ip: net.ParseIP("192.0.2.1"), rtt: time.Millisecond}, + {timedOut: true}, + {ip: net.ParseIP("192.0.2.1"), rtt: 3 * time.Millisecond}, + }) + + assert.Equal(t, 3, route.Total) + assert.Equal(t, 1, route.Failed) + assert.InDelta(t, 100.0/3.0, route.Loss, 0.001) + assert.Equal(t, 1000.0, route.MinCost) + assert.Equal(t, 2000.0, route.AvgCost) + assert.Equal(t, 3000.0, route.MaxCost) + assert.InDelta(t, 1414.213, route.StdCost, 0.001) + assert.Equal(t, "*", route.Items[1].IP) +} diff --git a/traceroute/udp_linux.go b/traceroute/udp_linux.go new file mode 100644 index 00000000..4204829a --- /dev/null +++ b/traceroute/udp_linux.go @@ -0,0 +1,481 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build linux + +package traceroute + +import ( + "context" + "encoding/binary" + "errors" + "math" + "net" + "sync" + "time" + + "golang.org/x/net/icmp" + "golang.org/x/net/ipv4" +) + +type udpLeaseKey struct { + target [net.IPv4len]byte + targetPort uint16 + sourcePort uint16 +} + +type udpLeaseRegistry struct { + mu sync.Mutex + leases map[udpLeaseKey]time.Time + cleanupTimer *time.Timer +} + +type udpProbeSocket struct { + conn *net.UDPConn + packetConn *ipv4.PacketConn + key udpLeaseKey + sent bool + appBuffer [2048]byte + icmpBuffer [2048]byte +} + +type udpWaitResult struct { + reply probeReply + err error +} + +// UDPProber reuses one raw ICMP receiver across sequential UDP probes. Probe +// calls are serialized so replies cannot be consumed by the wrong caller. +type UDPProber struct { + mu sync.Mutex + icmp *icmp.PacketConn +} + +var udpLeases = udpLeaseRegistry{leases: make(map[udpLeaseKey]time.Time)} //nolint:gochecknoglobals + +func traceUDP(ctx context.Context, target net.IP, cfg options) (Result, error) { + icmpConn, err := icmp.ListenPacket("ip4:icmp", "0.0.0.0") + if err != nil { + return Result{}, err + } + defer icmpConn.Close() //nolint:errcheck + + result := Result{Routes: make([]*Route, 0, cfg.maxTTL)} + for ttl := 1; ttl <= cfg.maxTTL; ttl++ { + replies := make([]probeReply, 0, cfg.attempts) + hopReached := false + for attempt := 0; attempt < cfg.attempts; attempt++ { + probeID := (ttl-1)*cfg.attempts + attempt + 1 + reply, err := sendIsolatedUDPProbe(ctx, icmpConn, target, cfg.port, ttl, probeID, cfg.timeout) + if err != nil { + if len(replies) > 0 { + result.Routes = append(result.Routes, repliesToRoute(replies)) + } + return result, err + } + replies = append(replies, reply) + hopReached = hopReached || reply.reached + } + result.Routes = append(result.Routes, repliesToRoute(replies)) + if hopReached { + result.Reached = true + return result, nil + } + } + return result, nil +} + +// ProbeUDP sends one IPv4 UDP probe. A TTL of zero keeps the operating +// system's default TTL, which is useful for endpoint reachability checks. +func ProbeUDP(ctx context.Context, target net.IP, port uint16, ttl int, + timeout time.Duration, +) (ProbeResult, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return ProbeResult{}, err + } + target, timeout, err := validateUDPProbe(target, port, ttl, timeout) + if err != nil { + return ProbeResult{}, err + } + prober, err := NewUDPProber() + if err != nil { + return ProbeResult{}, err + } + defer prober.Close() //nolint:errcheck + return prober.probe(ctx, target, port, ttl, timeout) +} + +// NewUDPProber opens a reusable raw ICMP receiver for UDP probes. +func NewUDPProber() (*UDPProber, error) { + conn, err := icmp.ListenPacket("ip4:icmp", "0.0.0.0") + if err != nil { + return nil, err + } + return &UDPProber{icmp: conn}, nil +} + +// Close closes the reusable raw ICMP receiver. +func (prober *UDPProber) Close() error { + if prober == nil { + return nil + } + prober.mu.Lock() + defer prober.mu.Unlock() + if prober.icmp == nil { + return nil + } + conn := prober.icmp + prober.icmp = nil + return conn.Close() +} + +// Probe sends one IPv4 UDP probe. A TTL of zero keeps the operating system's +// default TTL, which is useful for endpoint reachability checks. +func (prober *UDPProber) Probe(ctx context.Context, target net.IP, port uint16, ttl int, + timeout time.Duration, +) (ProbeResult, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return ProbeResult{}, err + } + target, timeout, err := validateUDPProbe(target, port, ttl, timeout) + if err != nil { + return ProbeResult{}, err + } + return prober.probe(ctx, target, port, ttl, timeout) +} + +func (prober *UDPProber) probe(ctx context.Context, target net.IP, port uint16, ttl int, + timeout time.Duration, +) (ProbeResult, error) { + if prober == nil { + return ProbeResult{}, errors.New("UDP prober is closed") + } + prober.mu.Lock() + defer prober.mu.Unlock() + if prober.icmp == nil { + return ProbeResult{}, errors.New("UDP prober is closed") + } + if ctx == nil { + ctx = context.Background() + } + socket, err := openUDPProbeSocket(ctx, target, port) + if err != nil { + return ProbeResult{}, err + } + defer socket.close() + reply, err := sendUDPProbe(ctx, prober.icmp, socket, target, port, ttl, 1, timeout) + result := ProbeResult{ + RTT: reply.rtt, + Reached: reply.reached, + TimedOut: reply.timedOut, + Sent: reply.sent, + } + if reply.ip != nil { + result.IP = reply.ip.String() + } + return result, err +} + +func validateUDPProbe(target net.IP, port uint16, ttl int, + timeout time.Duration, +) (net.IP, time.Duration, error) { + target = target.To4() + if target == nil { + return nil, 0, errors.New("udp probe target must be IPv4") + } + if target.IsUnspecified() || target.IsMulticast() || target.Equal(net.IPv4bcast) { + return nil, 0, errors.New("udp probe target must be a unicast address") + } + if port == 0 { + return nil, 0, errors.New("udp probe requires a destination port") + } + if ttl < 0 || ttl > MaxUDPHops { + return nil, 0, errors.New("udp probe TTL is out of range") + } + if timeout <= 0 { + timeout = defaultTimeout + } else if timeout > MaxTimeout { + timeout = MaxTimeout + } + return target, timeout, nil +} + +func openUDPProbeSocket(ctx context.Context, target net.IP, targetPort uint16) (*udpProbeSocket, error) { + var targetBytes [net.IPv4len]byte + copy(targetBytes[:], target.To4()) + for attempt := 0; attempt < 128; attempt++ { + if err := ctx.Err(); err != nil { + return nil, err + } + conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero}) + if err != nil { + return nil, err + } + addr, ok := conn.LocalAddr().(*net.UDPAddr) + if !ok || addr.Port <= 0 || addr.Port > math.MaxUint16 { + _ = conn.Close() + return nil, errors.New("get UDP traceroute source port") + } + key := udpLeaseKey{target: targetBytes, targetPort: targetPort, sourcePort: uint16(addr.Port)} + if !reserveUDPLease(key, time.Now()) { + _ = conn.Close() + continue + } + return &udpProbeSocket{conn: conn, packetConn: ipv4.NewPacketConn(conn), key: key}, nil + } + return nil, errors.New("allocate isolated UDP traceroute source port") +} + +func reserveUDPLease(key udpLeaseKey, now time.Time) bool { + return udpLeases.reserve(key, now) +} + +func (registry *udpLeaseRegistry) reserve(key udpLeaseKey, now time.Time) bool { + registry.mu.Lock() + defer registry.mu.Unlock() + if expires, ok := registry.leases[key]; ok && (expires.IsZero() || now.Before(expires)) { + return false + } + registry.leases[key] = time.Time{} + return true +} + +func (socket *udpProbeSocket) close() { + udpLeases.finish(socket.key, socket.sent, time.Now()) + _ = socket.conn.Close() +} + +func (registry *udpLeaseRegistry) finish(key udpLeaseKey, sent bool, now time.Time) { + registry.mu.Lock() + defer registry.mu.Unlock() + if sent { + registry.leases[key] = now.Add(MaxTimeout) + registry.scheduleCleanupLocked() + } else { + delete(registry.leases, key) + } +} + +func (registry *udpLeaseRegistry) scheduleCleanupLocked() { + if registry.cleanupTimer != nil { + return + } + registry.cleanupTimer = time.AfterFunc(MaxTimeout, func() { + registry.cleanupExpiredAt(time.Now()) + }) +} + +func (registry *udpLeaseRegistry) cleanupExpiredAt(now time.Time) { + registry.mu.Lock() + defer registry.mu.Unlock() + registry.cleanupTimer = nil + hasQuarantine := false + for key, expires := range registry.leases { + if expires.IsZero() { + continue + } + if !now.Before(expires) { + delete(registry.leases, key) + continue + } + hasQuarantine = true + } + if hasQuarantine { + registry.scheduleCleanupLocked() + } +} + +func sendIsolatedUDPProbe(ctx context.Context, icmpConn *icmp.PacketConn, target net.IP, + targetPort uint16, ttl, probeID int, timeout time.Duration, +) (probeReply, error) { + socket, err := openUDPProbeSocket(ctx, target, targetPort) + if err != nil { + return probeReply{}, err + } + defer socket.close() + return sendUDPProbe(ctx, icmpConn, socket, target, targetPort, ttl, probeID, timeout) +} + +func sendUDPProbe(ctx context.Context, icmpConn *icmp.PacketConn, socket *udpProbeSocket, + target net.IP, targetPort uint16, ttl, probeID int, timeout time.Duration, +) (probeReply, error) { + if err := ctx.Err(); err != nil { + return probeReply{}, err + } + if ttl > 0 { + if err := socket.packetConn.SetTTL(ttl); err != nil { + return probeReply{}, err + } + } + payload, udpLength, err := udpPayload(probeID) + if err != nil { + return probeReply{}, err + } + startedAt := time.Now() + if _, err := socket.conn.WriteToUDP(payload, &net.UDPAddr{IP: target, Port: int(targetPort)}); err != nil { + return probeReply{}, err + } + socket.sent = true + reply, err := waitUDPReply(ctx, icmpConn, socket.conn, target, socket.key.sourcePort, + targetPort, udpLength, startedAt, timeout, socket) + reply.sent = true + return reply, err +} + +func udpPayload(probeID int) ([]byte, uint16, error) { + const udpHeaderLength = 8 + if probeID <= 0 || probeID > MaxUDPProbePayload { + return nil, 0, errors.New("invalid UDP traceroute probe ID") + } + payload := make([]byte, probeID) + return payload, uint16(udpHeaderLength + len(payload)), nil +} + +func waitUDPReply(ctx context.Context, icmpConn *icmp.PacketConn, udpConn *net.UDPConn, + target net.IP, sourcePort, targetPort, udpLength uint16, startedAt time.Time, + timeout time.Duration, socket *udpProbeSocket, +) (probeReply, error) { + waitCtx, cancel := context.WithCancel(ctx) + results := make(chan udpWaitResult, 2) + go func() { + reply, err := waitUDPICMPReplyBuffer(waitCtx, icmpConn, socket.icmpBuffer[:], + target, sourcePort, targetPort, udpLength, startedAt, timeout) + results <- udpWaitResult{reply: reply, err: err} + }() + go func() { + reply, err := waitUDPApplicationReplyBuffer(waitCtx, udpConn, socket.appBuffer[:], + target, targetPort, startedAt, timeout) + results <- udpWaitResult{reply: reply, err: err} + }() + + var firstErr error + for pending := 2; pending > 0; pending-- { + result := <-results + if result.err == nil && !result.reply.timedOut { + cancel() + _ = icmpConn.SetReadDeadline(time.Now()) + _ = udpConn.SetReadDeadline(time.Now()) + for pending--; pending > 0; pending-- { + <-results + } + return result.reply, nil + } + if result.err != nil && firstErr == nil { + firstErr = result.err + } + } + cancel() + if err := ctx.Err(); err != nil { + return probeReply{}, err + } + if firstErr != nil { + return probeReply{}, firstErr + } + return probeReply{timedOut: true}, nil +} + +func waitUDPApplicationReply(ctx context.Context, conn *net.UDPConn, target net.IP, + targetPort uint16, startedAt time.Time, timeout time.Duration, +) (probeReply, error) { + return waitUDPApplicationReplyBuffer(ctx, conn, make([]byte, 2048), target, + targetPort, startedAt, timeout) +} + +func waitUDPApplicationReplyBuffer(ctx context.Context, conn *net.UDPConn, buffer []byte, + target net.IP, targetPort uint16, startedAt time.Time, timeout time.Duration, +) (probeReply, error) { + deadline := probeDeadline(ctx, startedAt, timeout) + for { + if err := conn.SetReadDeadline(probeReadDeadline(ctx, deadline)); err != nil { + return probeReply{}, err + } + _, peer, err := conn.ReadFromUDP(buffer) + if err != nil { + if ctx.Err() != nil { + return probeReply{}, ctx.Err() + } + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + if time.Now().Before(deadline) { + continue + } + return probeReply{timedOut: true}, nil + } + return probeReply{}, err + } + if peer == nil || peer.Port != int(targetPort) || !peer.IP.Equal(target) { + continue + } + return probeReply{ip: peer.IP, rtt: time.Since(startedAt), reached: true}, nil + } +} + +func waitUDPICMPReplyBuffer(ctx context.Context, conn *icmp.PacketConn, buffer []byte, + target net.IP, sourcePort, targetPort, udpLength uint16, startedAt time.Time, + timeout time.Duration, +) (probeReply, error) { + deadline := probeDeadline(ctx, startedAt, timeout) + for { + if err := conn.SetReadDeadline(probeReadDeadline(ctx, deadline)); err != nil { + return probeReply{}, err + } + n, peer, err := conn.ReadFrom(buffer) + if err != nil { + if ctx.Err() != nil { + return probeReply{}, ctx.Err() + } + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + if time.Now().Before(deadline) { + continue + } + return probeReply{timedOut: true}, nil + } + return probeReply{}, err + } + reply, matched := parseUDPICMPReply(buffer[:n], peer, target, sourcePort, targetPort, + udpLength, time.Since(startedAt)) + if matched { + return reply, nil + } + } +} + +func parseUDPICMPReply(packet []byte, peer net.Addr, target net.IP, + sourcePort, targetPort, udpLength uint16, rtt time.Duration, +) (probeReply, bool) { + message, err := icmp.ParseMessage(1, packet) + if err != nil { + return probeReply{}, false + } + quoted := quotedDatagram(message) + if quoted == nil { + return probeReply{}, false + } + header, err := ipv4.ParseHeader(quoted) + if err != nil || header.Protocol != 17 || len(quoted) < header.Len+8 || !header.Dst.Equal(target) { + return probeReply{}, false + } + udpHeader := quoted[header.Len : header.Len+8] + if binary.BigEndian.Uint16(udpHeader[0:2]) != sourcePort || + binary.BigEndian.Uint16(udpHeader[2:4]) != targetPort || + binary.BigEndian.Uint16(udpHeader[4:6]) != udpLength { + return probeReply{}, false + } + from, ok := ipFromAddr(peer) + if !ok { + return probeReply{}, false + } + _, destinationUnreachable := message.Body.(*icmp.DstUnreach) + return probeReply{ + ip: from, + rtt: rtt, + reached: destinationUnreachable && from.Equal(target), + }, true +} diff --git a/traceroute/udp_other.go b/traceroute/udp_other.go new file mode 100644 index 00000000..01a610a0 --- /dev/null +++ b/traceroute/udp_other.go @@ -0,0 +1,40 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build !linux && !windows + +package traceroute + +import ( + "context" + "errors" + "net" + "time" +) + +// UDPProber is unavailable outside Linux. +type UDPProber struct{} + +func traceUDP(context.Context, net.IP, options) (Result, error) { + return Result{}, errors.New("UDP traceroute is only supported on linux") +} + +// ProbeUDP is unavailable outside Linux. +func ProbeUDP(context.Context, net.IP, uint16, int, time.Duration) (ProbeResult, error) { + return ProbeResult{}, errors.New("UDP probe is only supported on linux") +} + +// NewUDPProber is unavailable outside Linux. +func NewUDPProber() (*UDPProber, error) { + return nil, errors.New("UDP probe is only supported on linux") +} + +// Probe is unavailable outside Linux. +func (*UDPProber) Probe(context.Context, net.IP, uint16, int, time.Duration) (ProbeResult, error) { + return ProbeResult{}, errors.New("UDP probe is only supported on linux") +} + +// Close is a no-op outside Linux. +func (*UDPProber) Close() error { return nil }