puzzle of the nff-go performance

[imroadsign]

im testing a dns server using nff-go ,code as below, but the performace is very slow,only 3,700 qps . device info : cpu core: 4 mem : 4G net card :1 dns test tool: dnsperf

im so confused, thx a lot if u can give me a reply.

---------code here------------ package main

import ( "flag" "github.com/google/gopacket" "github.com/google/gopacket/layers" "github.com/intel-go/nff-go/flow" "github.com/intel-go/nff-go/packet" "github.com/intel-go/nff-go/types" "github.com/miekg/dns" "net" "net/http" _ "net/http/pprof" )

// Parser and pre-allocated layers to be parsed. type gopacketContext2 struct { eth layers.Ethernet ip4 layers.IPv4 ip6 layers.IPv6 tcp layers.TCP udp layers.UDP dns layers.DNS parser *gopacket.DecodingLayerParser }

// Each handler will use its own gopacketContext. func (ctx gopacketContext2) Copy() interface{} { newCtx := new(gopacketContext2) newCtx.parser = gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, &newCtx.eth, &newCtx.ip4, &newCtx.ip6, &newCtx.tcp, &newCtx.udp,&newCtx.dns) return newCtx }

func (ctx gopacketContext2) Delete() { }

func main() { port := flag.Int("port", 0, "output port") inport := uint16(*port)

flow.CheckFatal(flow.SystemInit(nil))

inputFlow, _ := flow.SetReceiver(inport)

flow.CheckFatal(flow.SetIPForPort(inport, types.IPv4Address(200)<<24|types.IPv4Address(56)<<16|types.IPv4Address(168)<<8|types.IPv4Address(192)))

flow.CheckFatal(flow.DealARPICMP(inputFlow))

var ctx gopacketContext2

flow.SetHandler(inputFlow,gopacketHandleFunc2,ctx)

go http.ListenAndServe("0.0.0.0:3999", nil)
flow.CheckFatal(flow.SetStopper(inputFlow))

flow.CheckFatal(flow.SystemStart())

}

func gopacketHandleFunc2(current *packet.Packet, context flow.UserContext) {

if current.GetUDPNoCheck()!=nil && packet.SwapBytesUint16(current.GetUDPNoCheck().DstPort) == 53 {

    current.ParseL3()

    ctx := context.(*gopacketContext2)
    parser := ctx.parser
    decoded := []gopacket.LayerType{}

    packetData := current.GetRawPacketBytes()
    parser.DecodeLayers(packetData, &decoded)

    answerPacket, _ := packet.NewPacket()

    bts := printLayersInfo2(ctx, decoded)

    packet.InitEmptyIPv4UDPPacket(answerPacket, uint(len(bts)))

    if answerPacket != nil{

        answerPacket.Ether.DAddr = current.Ether.SAddr
        answerPacket.Ether.SAddr = current.Ether.DAddr
        ipv4 := current.GetIPv4()
        // Fill up L3
        (answerPacket.GetIPv4NoCheck()).DstAddr = ipv4.SrcAddr
        (answerPacket.GetIPv4NoCheck()).SrcAddr = ipv4.DstAddr

        // Fill up L4
        answerPacket.GetUDPNoCheck().DstPort = current.GetUDPNoCheck().SrcPort
        answerPacket.GetUDPNoCheck().SrcPort = current.GetUDPNoCheck().DstPort

        payloadBuffer,_ := answerPacket.GetPacketPayload()
        copy(payloadBuffer,bts)

        cks1 := packet.SwapBytesUint16(packet.CalculateIPv4Checksum(answerPacket.GetIPv4NoCheck()))

        cks2 := packet.SwapBytesUint16(packet.CalculateIPv4UDPChecksum(answerPacket.GetIPv4NoCheck(), answerPacket.GetUDPForIPv4(), answerPacket.Data))

        //fmt.Sprintf("%x -- %x",cks1, cks2)

        answerPacket.GetIPv4().HdrChecksum = cks1
        answerPacket.GetUDPNoCheck().DgramCksum = cks2

// fmt.Println(hex.Dump(answerPacket.GetRawPacketBytes()),"answerpacket")

        answerPacket.SendPacket(0)

    }

}

}

func printLayersInfo2(ctx *gopacketContext2, decoded []gopacket.LayerType)([]byte) {

answerRR := []layers.DNSResourceRecord{
    {
        Name:  ctx.dns.Questions[0].Name,
        Type:  layers.DNSTypeA,
        Class: layers.DNSClassIN,
        TTL:   1024,
        IP:    net.IP([]byte{1, 2, 3, 4}),
    },
}

ctx.dns.QR = true
ctx.dns.Answers = answerRR
ctx.dns.Additionals = nil
ctx.dns.ARCount = 0
ctx.dns.TC = false
ctx.dns.AA = false
ctx.dns.RD = true
ctx.dns.RA = true
ctx.dns.OpCode = dns.OpcodeQuery
ctx.dns.ResponseCode = layers.DNSResponseCodeNoErr

//fmt.Println(ctx.dns)

buff := gopacket.NewSerializeBuffer()

opt := gopacket.SerializeOptions{
    FixLengths : true,
    ComputeChecksums :true,
}

ctx.dns.SerializeTo(buff,opt)

return buff.Bytes()

}

---------------go profile here --------

roadsign@ubuntu:~$ go tool pprof http://localhost:3999/debug/pprof/profile Fetching profile over HTTP from http://localhost:3999/debug/pprof/profile Saved profile in /home/roadsign/pprof/pprof.pingReplay.samples.cpu.003.pb.gz File: pingReplay Build ID: 33be144b95dc0b5aacd1622370aaaf75728a768f Type: cpu Time: Jul 16, 2020 at 2:15am (PDT) Duration: 30.18s, Total samples = 1.40mins (277.40%) Entering interactive mode (type "help" for commands, "o" for options) (pprof) top Showing nodes accounting for 81.37s, 97.20% of 83.71s total Dropped 172 nodes (cum <= 0.42s) Showing top 10 nodes out of 38 flat flat% sum% cum cum% 74.60s 89.12% 89.12% 74.63s 89.15% runtime.cgocall 2.15s 2.57% 91.69% 4.81s 5.75% runtime.selectgo 1.09s 1.30% 92.99% 1.09s 1.30% runtime.madvise 0.86s 1.03% 94.02% 0.86s 1.03% runtime.lock 0.77s 0.92% 94.93% 0.80s 0.96% runtime.unlock 0.66s 0.79% 95.72% 0.66s 0.79% runtime.futex 0.34s 0.41% 96.13% 27.46s 32.80% github.com/intel-go/nff-go/flow.segmentProcess 0.34s 0.41% 96.54% 1.18s 1.41% runtime.sellock 0.29s 0.35% 96.88% 1.05s 1.25% runtime.selunlock 0.27s 0.32% 97.20% 0.48s 0.57% runtime.scanblock (pprof) tree Showing nodes accounting for 81.41s, 97.25% of 83.71s total Dropped 172 nodes (cum <= 0.42s) ----------------------------------------------------------+------------- flat flat% sum% cum cum% calls calls% + context
----------------------------------------------------------+------------- 27.46s 36.79% | github.com/intel-go/nff-go/internal/low._Cfunc_receiveRSS 25.95s 34.77% | github.com/intel-go/nff-go/internal/low._Cfunc_nff_go_stop 21.12s 28.30% | github.com/intel-go/nff-go/internal/low._Cfunc_directSend 0.10s 0.13% | main.gopacketHandleFunc2 74.60s 89.12% 89.12% 74.63s 89.15% | runtime.cgocall ----------------------------------------------------------+------------- 4.81s 100% | github.com/intel-go/nff-go/flow.segmentProcess 2.15s 2.57% 91.69% 4.81s 5.75% | runtime.selectgo 1.18s 24.53% | runtime.sellock 1.05s 21.83% | runtime.selunlock ----------------------------------------------------------+------------- 1.09s 100% | runtime.sysUnused 1.09s 1.30% 92.99% 1.09s 1.30% | runtime.madvise ----------------------------------------------------------+------------- 0.84s 97.67% | runtime.sellock 0.86s 1.03% 94.02% 0.86s 1.03% | runtime.lock ----------------------------------------------------------+------------- 0.76s 95.00% | runtime.selunlock 0.77s 0.92% 94.93% 0.80s 0.96% | runtime.unlock ----------------------------------------------------------+------------- 0.52s 78.79% | runtime.futexwakeup 0.66s 0.79% 95.72% 0.66s 0.79% | runtime.futex ----------------------------------------------------------+------------- 27.46s 100% | github.com/intel-go/nff-go/flow.(instance).startNewClone.func1 0.34s 0.41% 96.13% 27.46s 32.80% | github.com/intel-go/nff-go/flow.segmentProcess 22.14s 80.63% | github.com/intel-go/nff-go/flow.handle 4.81s 17.52% | runtime.selectgo ----------------------------------------------------------+------------- 1.18s 100% | runtime.selectgo 0.34s 0.41% 96.54% 1.18s 1.41% | runtime.sellock 0.84s 71.19% | runtime.lock ----------------------------------------------------------+------------- 1.05s 100% | runtime.selectgo 0.29s 0.35% 96.88% 1.05s 1.25% | runtime.selunlock 0.76s 72.38% | runtime.unlock ----------------------------------------------------------+------------- 0.48s 100% | runtime.markrootBlock 0.27s 0.32% 97.20% 0.48s 0.57% | runtime.scanblock ----------------------------------------------------------+------------- 22.14s 100% | github.com/intel-go/nff-go/flow.handle 0.03s 0.036% 97.24% 22.14s 26.45% | main.gopacketHandleFunc2 21.13s 95.44% | github.com/intel-go/nff-go/packet.(Packet).SendPacket 0.10s 0.45% | runtime.cgocall

----------------------dns perf report here--------------

Statistics:

Queries sent: 1535994 Queries completed: 1535771 (99.99%) Queries lost: 180 (0.01%) Queries interrupted: 43 (0.00%)

Response codes: NOERROR 1535771 (100.00%) Average packet size: request 32, response 62 Run time (s): 48.760166 Queries per second: 31496.426817

[imroadsign2]

for help~~~

[aregm]

Hi, @imroadsign2! Thanks for submitting the question. There are two main issues with the example - you use DPDK based Go sender which does several copies of the packet when you mix it with the gopacket, which is OS based. And NFF-Go is created to use at least 4 cores. I see a throughput of 30K which is not bad for a laptop, taking into account the number of packet copies done and the fact that you use only one core. And I suppose you are using VM, which is adding to the problem. Please run any of the examples and let me know if you see better performance, without using gopacket.

[aregm]

@imroadsign2 - Can I close the issue?