add "raw socket" Data AcQuisition sniffing source for the BSDs

[david415]

Do not force *BSD users to use libpcap. Instead use the operating systems native system calls for sniffing packets. Raw sockets are the first thing that comes to mind... but perhaps there is another way?

[mischief]

it looks like openbsd, freebsd, and osx support IPPROTO_DIVERT, but i haven't found anything for netbsd.

[david415]

Sounds good... i need a vm running a bsd to test with i think...

[mischief]

netbsd only supports bpf apparently.

[david415]

perhaps if we look around we'll find a good C code bpf written for NetBSD... and we can use it as inspiration to write one in pure golang with a bit of cgo references for the ioctl syscalls?

or is all that unneeded because: https://golang.org/src/syscall/bpf_bsd.go

[mischief]

yes, it looks like you could use that api.

[david415]

yes and it looks like RawConn.ReadFrom might work on all platforms https://godoc.org/golang.org/x/net/ipv4#RawConn.ReadFrom

perhaps initially there should be three DAQs:

• libpcap
• AF_PACKET (linux only)
• RawConn (ipv4 only)

[david415]

presumably this tiny example sniffer will run on openbsd... wanna try it?:

package main

import (
    "fmt"
    "net"
)

func main() {
    protocol := "icmp"
    netaddr, _ := net.ResolveIPAddr("ip4", "127.0.0.1")
    conn, _ := net.ListenIP("ip4:"+protocol, netaddr)

    buf := make([]byte, 1024)
    numRead, _, _ := conn.ReadFrom(buf)
    fmt.Printf("% X\n", buf[:numRead])
}

[mischief]

it works with icmp but not tcp.

[david415]

i've been trying and failing to get this other method to work:

package main

import (
    "fmt"
    "golang.org/x/net/ipv4"
    "net"
)

func main() {
    var packetConn net.PacketConn
    var rawConn *ipv4.RawConn
    var err error
    var header *ipv4.Header = new(ipv4.Header)
    var cm *ipv4.ControlMessage
    var buf []byte

    packetConn, err = net.ListenPacket("ip4:tcp", "127.0.0.1")
    if err != nil {
        panic(err)
    }
    rawConn, err = ipv4.NewRawConn(packetConn)
    if err != nil {
        panic(err)
    }
    header, buf, cm, err = rawConn.ReadFrom(buf)
    if err != nil {
        panic(err)
    }
    fmt.Printf("cm %s\n", cm)
    fmt.Printf("buf %s\n", buf)
    fmt.Printf("header %s\n", header)
}

[david415]

ok... i guess these RawConn and PacketConn aren't going to work on NetBSD and OpenBSD either... so presumably the only way to sniff if to use that bpf_bsd.go API?

I'm not exactly sure how to use it, however most of the API calls here want a filedescriptor... aka socket. Are we supposed to use cgo to create a raw socket and then feed the api calls that fd?

[david415]

ok here's my working prototype ethernet sniffer for *BSDs... however the bpf_hdr is slightly different on each BSD and sometimes depending on architecture... i got this to work on my FreeBSD vps. I borrowed a tiny bit of code from https://github.com/songgao/ether

package main

import (
    "fmt"
    "github.com/google/gopacket"
    "github.com/google/gopacket/layers"
    "syscall"
    "unsafe"
)

const wordSize = int(unsafe.Sizeof(uintptr(0)))

// FreeBSD style bpf_hdr
type bpf_hdr struct {
    bh_tstamp  syscall.Timeval // 8 or 16 bytes depending on arch
    bh_caplen  uint32
    bh_datalen uint32
    bh_hdrlen  uint16
}

func bpf_wordalign(x int) int {
    return (((x) + (wordSize - 1)) &^ (wordSize - 1))
}

type BpfSniffer struct {
    fd       int
    name     string
    stopChan chan bool
    readChan chan []byte
}

func NewBpfSniffer() *BpfSniffer {
    return &BpfSniffer{
        stopChan: make(chan bool, 0),
        readChan: make(chan []byte, 0),
    }
}

func (b *BpfSniffer) Init(name string) error {
    var err error
    enable := 1

    for i := 0; i < 99; i++ {
        b.fd, err = syscall.Open("/dev/bpf0", syscall.O_RDWR, 0) // XXX 0
        if err == nil {
            break
        }
    }

    b.name = name
    err = syscall.SetBpfInterface(b.fd, b.name)
    if err != nil {
        return err
    }
    err = syscall.SetBpfImmediate(b.fd, enable)
    if err != nil {
        return err
    }
    err = syscall.SetBpfHeadercmpl(b.fd, enable)
    if err != nil {
        return err
    }
    err = syscall.SetBpfPromisc(b.fd, enable)
    if err != nil {
        return err
    }

    go b.readPackets()
    return nil
}

func (b *BpfSniffer) Stop() {
    b.stopChan <- true
}

func (b *BpfSniffer) readPackets() {

    bufLen, err := syscall.BpfBuflen(b.fd)
    if err != nil {
        panic(err)
    }
    buf := make([]byte, bufLen)
    var n int

    for {
        select {
        case <-b.stopChan:
            return
        default:
            n, err = syscall.Read(b.fd, buf)
            if err != nil {
                return
            }
            p := int(0)
            for p < n {
                hdr := (*bpf_hdr)(unsafe.Pointer(&buf[p]))
                frameStart := p + int(hdr.bh_hdrlen)
                b.readChan <- buf[frameStart : frameStart+int(hdr.bh_caplen)]
                p += bpf_wordalign(int(hdr.bh_hdrlen) + int(hdr.bh_caplen))
            }
        }
    }
}

func (b *BpfSniffer) ReadFrame() []byte {
    frame := <-b.readChan
    return frame
}

func main() {
    var err error
    sniffer := NewBpfSniffer()
    err = sniffer.Init("vtnet0")
    if err != nil {
        panic(err)
    }

    for {
        buf := sniffer.ReadFrame()
        // Decode a packet
        packet := gopacket.NewPacket(buf, layers.LayerTypeEthernet, gopacket.Default)
        // Get the TCP layer from this packet
        if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
            fmt.Println("This is a TCP packet!")
            // Get actual TCP data from this layer
            tcp, _ := tcpLayer.(*layers.TCP)
            fmt.Printf("From src port %d to dst port %d\n", tcp.SrcPort, tcp.DstPort)
        }

        // Iterate over all layers, printing out each layer type
        for _, layer := range packet.Layers() {
            fmt.Println("PACKET LAYER:", layer.LayerType())
        }
    }
}

[david415]

boom! code review please!

This works on FreeBSD... and probably the rest of the *BSDs:

https://github.com/david415/HoneyBadger/tree/71.add-bsd-sniffer.1 https://github.com/david415/HoneyBadger/commit/1a4e2df5e781f0d90289daf96ebbae0cbb814163 https://github.com/david415/HoneyBadger/tree/71.add-bsd-sniffer.1/cmd/testBpfSniffer

[mischief]

@david415 i think unsafe.Sizeof(uintptr) gives you the native word size.

the fact that the bsd system headers don't use exact width types annoys me greatly.

[david415]

yes you were correct about the word size... i had to create several optional compile files for the bpf_hdr golang struct... i borrowed the darwin compatibility from that guy's ether repository... and i made a default BSD file... but I only tested it on FreeBSD so far.

also... i had to make dummy AF_PACKET module for BSDs... and dummy BPF module for Linux... in order to get it to build on Linux and FreeBSD.

https://github.com/david415/HoneyBadger/tree/71.add-bsd-sniffer.1 https://github.com/david415/HoneyBadger/commit/2db835f0591421896bba2c57c29f9fc95dfed7a1

[david415]

ouch... i do believe it is better to use the external syscall api rather than the internal one... here we see the bpf header struct defined for OpenBSD on am64:

https://github.com/golang/sys/blob/master/unix/ztypes_openbsd_amd64.go#L427

and another one for NetBSD: https://github.com/golang/sys/blob/master/unix/ztypes_netbsd_amd64.go#L369

these recent additions to syscall look great... and we should use them. It seems to me that we can get rid of our own definition of BpfHdr and use the one in the syscall package... as long as the running system is a BSD.

[david415]

this issue is resolved in this commit https://github.com/david415/HoneyBadger/commit/2db835f0591421896bba2c57c29f9fc95dfed7a1