[Question]: first off, how to test the tls feature of gnet?

[kolinfluence]

Actions I've taken before I'm here

• [x] I've thoroughly read the documentations about this problem but still have no answer.
• [X] I've searched the Github Issues/Discussions but didn't find any similar problems that have been solved.
• [X] I've searched the internet for this problem but didn't find anything helpful.

Questions with details

as titled

Code snippets (optional)

No response

[leslie-fei]

https://github.com/leslie-fei/gnet/blob/584e828a0eed8705581fd5a839c97ea8c7d724bc/gnet_test.go#L42

Here is test code

[kolinfluence]

i did

go get github.com/panjf2000/gnet/v2@dev

is this repo correct?

i would like to test it this way as main.go file,

go build httpsgnet.go 
# command-line-arguments
./httpsgnet.go:23:51: undefined: gnet.Server
./httpsgnet.go:58:8: undefined: gnet.WithCodec
./httpsgnet.go:58:22: undefined: HttpCodec
./httpsgnet.go:59:8: undefined: gnet.WithTLS

(chatgpt4 suggested)

possible to make a https version for testing? else a tcp echo version will do fine too

package main

import (
        "crypto/tls"
        "flag"
        "fmt"
        "log"
        "time"

        "github.com/panjf2000/gnet/v2"
        "github.com/panjf2000/gnet/v2/pkg/pool/goroutine"
)

type httpsServer struct {
        gnet.BuiltinEventEngine

        addr      string
        multicore bool
        eng       gnet.Engine
        pool      *goroutine.Pool
}

func (hs *httpsServer) OnInitComplete(server gnet.Server) (action gnet.Action) {
        log.Printf("HTTPS server is listening on %s (multi-core: %t, loops: %d)\n", hs.addr, hs.multicore, server.NumEventLoop)
        return
}

func (hs *httpsServer) OnOpened(c gnet.Conn) ([]byte, gnet.Action) {
        return []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 12\r\n\r\nHello world!"), gnet.Close
}

func (hs *httpsServer) OnClosed(c gnet.Conn, err error) (action gnet.Action) {
        log.Printf("Closed connection on %s, error: %v", c.RemoteAddr().String(), err)
        return
}

func main() {
        var port int
        var multicore bool

        flag.IntVar(&port, "port", 443, "server port")
        flag.BoolVar(&multicore, "multicore", true, "multicore with multiple CPU cores")
        flag.Parse()

        addr := fmt.Sprintf("tcp://:%d", port)
        tlsConfig := &tls.Config{
                Certificates: []tls.Certificate{mustLoadCertificate()},
        }
        hs := &httpsServer{
                addr:      addr,
                multicore: multicore,
                pool:      goroutine.Default(),
        }

        options := []gnet.Option{
                gnet.WithMulticore(multicore),
                gnet.WithTCPKeepAlive(time.Minute * 5),
                gnet.WithCodec(new(HttpCodec)), // Define your own codec for HTTP handling.
                gnet.WithTLS(tlsConfig),
                gnet.WithReusePort(true),
        }

        log.Fatal(gnet.Run(hs, hs.addr, options...))
}

func mustLoadCertificate() tls.Certificate {
        cert, err := tls.LoadX509KeyPair("server.crt", "server.key")
        if err != nil {
                log.Fatalf("Failed to load server certificate: %v", err)
        }
        return cert
}

[leslie-fei]

where are you come from? I understand Chinese. If you prefer, I can communicate with you in Chinese.

[leslie-fei]

In the next few days, I will use your example to see if I can send and receive data packets over HTTPS.

[leslie-fei]

@kolinfluence here is test code for TLS

package main

import (
    tls2 "crypto/tls"
    "flag"
    "fmt"
    "io"
    "log"
    "net/http"
    "time"

    "github.com/panjf2000/gnet/v2"
    "github.com/panjf2000/gnet/v2/pkg/pool/goroutine"
    "github.com/panjf2000/gnet/v2/pkg/tls"
)

type httpsServer struct {
    gnet.BuiltinEventEngine

    addr      string
    multicore bool
    eng       gnet.Engine
    pool      *goroutine.Pool
}

func (hs *httpsServer) OnInitComplete(server gnet.Conn) (action gnet.Action) {
    log.Printf("HTTPS server is listening on %s (multi-core: %t, loops: %d)\n", hs.addr, hs.multicore, 0)
    return
}

func (hs *httpsServer) OnOpen(c gnet.Conn) ([]byte, gnet.Action) {
    return []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 12\r\n\r\nHello world!"), gnet.None
}

func (hs *httpsServer) OnClose(c gnet.Conn, err error) (action gnet.Action) {
    log.Printf("Closed connection on %s, error: %v", c.RemoteAddr().String(), err)
    return
}

func main() {
    var port int
    var multicore bool

    flag.IntVar(&port, "port", 443, "server port")
    flag.BoolVar(&multicore, "multicore", true, "multicore with multiple CPU cores")
    flag.Parse()

    addr := fmt.Sprintf("tcp://:%d", port)
    tlsConfig := &tls.Config{
        Certificates: []tls.Certificate{mustLoadCertificate()},
    }
    hs := &httpsServer{
        addr:      addr,
        multicore: multicore,
        pool:      goroutine.Default(),
    }

    options := []gnet.Option{
        gnet.WithMulticore(multicore),
        gnet.WithTCPKeepAlive(time.Minute * 5),
        // gnet.WithCodec(new(HttpCodec)), // Define your own codec for HTTP handling.
        gnet.WithTLSConfig(tlsConfig),
        gnet.WithReusePort(true),
    }

    // new a http client to call
    go func() {
        time.Sleep(time.Second)
        httpClient := &http.Client{
            Timeout: 5 * time.Second,
            Transport: &http.Transport{
                TLSClientConfig: &tls2.Config{
                    InsecureSkipVerify: true,
                },
            },
        }
        rsp, err := httpClient.Get("https://127.0.0.1")
        if err != nil {
            log.Printf("client call error: %v\n", err)
            return
        }
        defer rsp.Body.Close()
        data, err := io.ReadAll(rsp.Body)
        if err != nil {
            log.Fatalf("read data error: %v\n", err)
        }
        log.Printf("http client call success, code: %d, data: %s\n", rsp.StatusCode, data)
    }()

    log.Fatal(gnet.Run(hs, hs.addr, options...))
}

func mustLoadCertificate() tls.Certificate {
    cert, err := tls.LoadX509KeyPair("server.crt", "server.key")
    if err != nil {
        log.Fatalf("Failed to load server certificate: %v", err)
    }
    return cert
}

and the console output the expected results

you should execute git pull https://github.com/leslie-fei/gnet.git, then modify the go.mod file by replacing 'gnet' with the local path like this

replace github.com/panjf2000/gnet/v2 => ../gnet

require github.com/panjf2000/gnet/v2 v2.0.0-00010101000000-000000000000

require (
    github.com/panjf2000/ants/v2 v2.9.0 // indirect
    github.com/valyala/bytebufferpool v1.0.0 // indirect
    go.uber.org/atomic v1.7.0 // indirect
    go.uber.org/multierr v1.6.0 // indirect
    go.uber.org/zap v1.21.0 // indirect
    golang.org/x/crypto v0.21.0 // indirect
    golang.org/x/sync v0.6.0 // indirect
    golang.org/x/sys v0.18.0 // indirect
    gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
)

[kolinfluence]

@leslie-fei is there anything wrong the soreuseport option when using tls? previous working versions from https://github.com/0-haha/gnet_tls_examples cannot use so reuseport

[leslie-fei]

@leslie-fei is there anything wrong the soreuseport option when using tls? previous working versions from https://github.com/0-haha/gnet_tls_examples cannot use so reuseport

OK, I try it later

[kolinfluence]

@leslie-fei i think dont waste time try it, it uses old gnet and seemed the tls here also cannot use reuse port option https://github.com/luyu6056/tls https://github.com/luyu6056/gnet

if you can get it to work and know why, do "fix" it if it's not going to take more than 30 mins.

[kolinfluence]

@leslie-fei can you save the main.go file in gnet/example folder (or somewhere else) and run the main file?

this works on browser but i'm getting these information, not sure if it's my cert / key issue. not sure and i cant confirm. so can you make the example folder that works with a self signed cert / key?

[gnet] 2024-04-22T04:41:03.922476619-04:00  ERROR   logging/logger.go:241   error occurs during runtime, remote error: tls: unknown certificate
github.com/panjf2000/gnet/v2/pkg/logging.Error
    /usr/local/src/gnet_tls_examples/echo/server/lesismal/gnet/pkg/logging/logger.go:241
github.com/panjf2000/gnet/v2.(*tlsEventHandler).OnTraffic
    /usr/local/src/gnet_tls_examples/echo/server/lesismal/gnet/tls.go:179
github.com/panjf2000/gnet/v2.(*eventloop).read
    /usr/local/src/gnet_tls_examples/echo/server/lesismal/gnet/eventloop_unix.go:124
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /usr/local/src/gnet_tls_examples/echo/server/lesismal/gnet/reactor_default_linux.go:116
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /usr/local/src/gnet_tls_examples/echo/server/lesismal/gnet/internal/netpoll/epoll_default_poller.go:134
github.com/panjf2000/gnet/v2.(*eventloop).run
    /usr/local/src/gnet_tls_examples/echo/server/lesismal/gnet/reactor_default_linux.go:97
golang.org/x/sync/errgroup.(*Group).Go.func1
    /root/go/pkg/mod/golang.org/x/sync@v0.6.0/errgroup/errgroup.go:78
2024/04/22 04:41:03 Closed connection on 127.0.0.1:39646, error: <nil>
2024/04/22 04:41:03 Closed connection on 127.0.0.1:41990, error: read: connection reset by peer

[kolinfluence]

@leslie-fei it works with wget https://127.0.0.1:443/ --no-check-certificate but doesnt work on browser, the browser will show the previous error log i mentioned

and it doesnt work with wrk -d 30 -c 100 https://127.0.0.1:443/

u can try and see what i mean

[kolinfluence]

pls send me the link where i can buy u coffees for making this work on browser. thx

[leslie-fei]

@leslie-fei it works with wget https://127.0.0.1:443/ --no-check-certificate but doesnt work on browser, the browser will show the previous error log i mentioned

and it doesnt work with wrk -d 30 -c 100 https://127.0.0.1:443/

u can try and see what i mean

When I open the browser, it's fine to ignore certificate authentication. There should be an issue with wrk as well, probably not ignoring certificate validation. Do you have WeChat? Can I communicate with you through WeChat?

[leslie-fei]

@kolinfluence If you can add me on WeChat, my username is linfeipengwx.

[kolinfluence]

@leslie-fei ok, will add u.

1. if you use browser you'll see the error log i mentioned.

and if you use wrk,

2. i need to manually cancel the wrk

   wrk -c 100000 -d 30 http://127.0.0.1:443/

   Running 30s test @ http://127.0.0.1:443/
   2 threads and 100000 connections
   ^C  Thread Stats   Avg      Stdev     Max   +/- Stdev
   Latency     0.00us    0.00us   0.00us    -nan%
   Req/Sec     0.00      0.00     0.00      -nan%
   0 requests in 7.08s, 0.00B read
   Socket errors: connect 98981, read 0, write 0, timeout 0
   Requests/sec:      0.00
   Transfer/sec:       0.00B

[leslie-fei]

@leslie-fei ok, will add u.

1. if you use browser you'll see the error log i mentioned.

and if you use wrk,

2. i need to manually cancel the wrk

wrk -c 100000 -d 30 http://127.0.0.1:443/

Running 30s test @ http://127.0.0.1:443/
  2 threads and 100000 connections
^C  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency     0.00us    0.00us   0.00us    -nan%
    Req/Sec     0.00      0.00     0.00      -nan%
  0 requests in 7.08s, 0.00B read
  Socket errors: connect 98981, read 0, write 0, timeout 0
Requests/sec:      0.00
Transfer/sec:       0.00B

It looks like there is an issue, I need time to fix it.

[leslie-fei]

@kolinfluence

You can update the latest code, I have tested that REUSEPORT works. Below is the test code.

package main

import (
    "flag"
    "fmt"
    "log"
    "time"

    "github.com/panjf2000/gnet/v2"
    "github.com/panjf2000/gnet/v2/pkg/logging"
    "github.com/panjf2000/gnet/v2/pkg/pool/goroutine"
    "github.com/panjf2000/gnet/v2/pkg/tls"
)

func main() {
    runHTTPServer()
}

func runHTTPServer() {
    var port int
    var multicore bool

    flag.IntVar(&port, "port", 443, "server port")
    flag.BoolVar(&multicore, "multicore", true, "multicore with multiple CPU cores")
    flag.Parse()

    addr := fmt.Sprintf("tcp://:%d", port)
    tlsConfig := &tls.Config{
        Certificates: []tls.Certificate{mustLoadCertificate()},
    }
    hs := &httpsServer{
        addr:      addr,
        multicore: multicore,
        pool:      goroutine.Default(),
    }

    options := []gnet.Option{
        gnet.WithMulticore(multicore),
        gnet.WithTCPKeepAlive(time.Minute * 5),
        gnet.WithTLSConfig(tlsConfig),
        gnet.WithReusePort(true),
    }

    log.Fatal(gnet.Run(hs, hs.addr, options...))
}

type httpsServer struct {
    gnet.BuiltinEventEngine

    addr      string
    multicore bool
    eng       gnet.Engine
    pool      *goroutine.Pool
}

/*func (hs *httpsServer) OnOpen(c gnet.Conn) ([]byte, gnet.Action) {
    // logging.Infof("OnOpen addr: %s", c.RemoteAddr().String())
    return []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 12\r\n\r\nHello world!"), gnet.None
}*/

func (hs *httpsServer) OnTraffic(c gnet.Conn) (action gnet.Action) {
    // read all get http request
    _, _ = c.Next(-1)
    // TODO decode http codec
    // TODO handling http request and response content, should decode http request for yourself
    logging.Infof("OnTraffic...")
    // for example hello response
    _, _ = c.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 12\r\n\r\nHello world!"))
    return
}

func (hs *httpsServer) OnClosed(c gnet.Conn, err error) (action gnet.Action) {
    log.Printf("Closed connection on %s, error: %v", c.RemoteAddr().String(), err)
    return
}

func mustLoadCertificate() tls.Certificate {
    cert, err := tls.LoadX509KeyPair("server.crt", "server.key")
    if err != nil {
        log.Fatalf("Failed to load server certificate: %v", err)
    }
    return cert
}

REUSE and WRK

[kolinfluence]

@leslie-fei there's still some problem... wrk gets stuck sometimes. and the runs of wrk a few times shows very inconsistent results. sometimes 220,000 req /s, sometimes 75,000 req/s. the disparity is too huge.

GOMAXPROCS=1 ./main

it gets stuck sometimes. pls try a few times and see

wrk  -d 5 https://127.0.0.1:443/
Running 5s test @ https://127.0.0.1:443/
  3 threads and 10 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency     0.00us    0.00us   0.00us    -nan%
    Req/Sec     0.00      0.00     0.00      -nan%
  0 requests in 5.01s, 0.00B read
Requests/sec:      0.00
Transfer/sec:       0.00B

without GOMAXPROCS, sometimes it works, sometimes doesnt.

$ wrk -d 5 https://127.0.0.1:443/
Running 5s test @ https://127.0.0.1:443/
  2 threads and 10 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    22.48us   13.49us   2.17ms   99.69%
    Req/Sec    37.63k     6.78k   47.79k    58.00%
  373991 requests in 5.00s, 27.46MB read
Requests/sec:  74796.00
Transfer/sec:      5.49MB

$ wrk -d 5 -t 4 -c 100 https://127.0.0.1:443/
Running 5s test @ https://127.0.0.1:443/
  4 threads and 100 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency     0.00us    0.00us   0.00us    -nan%
    Req/Sec     0.00      0.00     0.00      -nan%
  0 requests in 5.08s, 0.00B read
Requests/sec:      0.00
Transfer/sec:       0.00B

[leslie-fei]

@kolinfluence I feel like the issue may be caused by not reading the complete HTTP packet. I didn't encounter the situation you described during multiple tests with wrk. Can you show me your wrk version? Please modify the example code and try again. Unstable QPS, possibly due to logging issues. Try disabling it in the example and test again.

func (hs *httpsServer) OnTraffic(c gnet.Conn) (action gnet.Action) {
    // read all get http request
    // TODO decode http codec
    // TODO handling http request and response content, should decode http request for yourself
    // Must read the complete HTTP packet before responding.
    if hs.isHTTPRequestComplete(c) {
        _, _ = c.Next(-1)
        // for example hello response
        _, _ = c.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 12\r\n\r\nHello world!"))
    }
    return
}

func (hs *httpsServer) isHTTPRequestComplete(c gnet.Conn) bool {
    buf, _ := c.Peek(c.InboundBuffered())
    return bytes.Contains(buf, []byte("\r\n\r\n"))
}

[kolinfluence]

@leslie-fei i am using ubuntu 22.04 amd ryzen 5 12core laptop

maybe u changed something i didnt update.

1. i already disabled logging.
2. can you provide another instruction to show how to get your version? i try to reinstall etc.
3. i will try using the mod version u mention and revert.

[leslie-fei]

@kolinfluence https://github.com/leslie-fei/gnet-example my test code git repository

[kolinfluence]

@leslie-fei tried, didnt work, wrk can stuck sometimes, the more you run, the performance drops greatly and wrk still does 0

i try on another laptop and revert.. hang on

[kolinfluence]

@leslie-fei on my other laptop, how to resolve?

root@ubuntu:/home/ubuntu/gnet# go mod init gnettls
go: creating new go.mod: module gnettls
go: to add module requirements and sums:
    go mod tidy
root@ubuntu:/home/ubuntu/gnet# go mod tidy
go: finding module for package github.com/panjf2000/gnet/v2
go: finding module for package github.com/panjf2000/gnet/v2/pkg/tls
go: finding module for package github.com/panjf2000/gnet/v2/pkg/pool/goroutine
go: found github.com/panjf2000/gnet/v2 in github.com/panjf2000/gnet/v2 v2.5.0
go: found github.com/panjf2000/gnet/v2/pkg/pool/goroutine in github.com/panjf2000/gnet/v2 v2.5.0
go: finding module for package github.com/panjf2000/gnet/v2/pkg/tls
go: gnettls imports
    github.com/panjf2000/gnet/v2/pkg/tls: module github.com/panjf2000/gnet/v2@latest found (v2.5.0), but does not contain package github.com/panjf2000/gnet/v2/pkg/tls

[leslie-fei]

@kolinfluence you can update the latest code, https://github.com/leslie-fei/gnet-example

[kolinfluence]

@leslie-fei i tried. can you do

GOMAXPROCS=1 ./main &

and try with

wrk -t 3 -d 10 https://127.0.0.1:443/

?

try running wrk a few times using the same parameter

[leslie-fei]

wrk -t 3 -d 10 https://127.0.0.1:443/

@kolinfluence

[root@localhost gnet-example]# GOMAXPROCS=1 ./main &
[1] 13015
[root@localhost gnet-example]# [gnet] 2024-04-24T02:04:46.19522445-04:00        INFO    logging/logger.go:256   Launching gnet with 8 event-loops, l                                                  istening on: tcp://:443

[root@localhost gnet-example]# wrk -t 3 -d 10 https://127.0.0.1:443/
Running 10s test @ https://127.0.0.1:443/
  3 threads and 10 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    83.64us  250.27us  17.99ms   99.19%
    Req/Sec    38.42k     5.46k   48.75k    68.54%
  1153909 requests in 10.10s, 84.73MB read
Requests/sec: 114257.14
Transfer/sec:      8.39MB
[root@localhost gnet-example]# wrk -t 3 -d 10 https://127.0.0.1:443/
Running 10s test @ https://127.0.0.1:443/
  3 threads and 10 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    66.40us   71.59us   6.76ms   96.99%
    Req/Sec    43.59k     5.12k   56.81k    73.93%
  1313587 requests in 10.10s, 96.46MB read
Requests/sec: 130057.60
Transfer/sec:      9.55MB
[root@localhost gnet-example]# wrk -t 3 -d 10 https://127.0.0.1:443/
Running 10s test @ https://127.0.0.1:443/
  3 threads and 10 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    73.29us  153.07us   9.07ms   98.37%
    Req/Sec    44.20k     6.30k   55.83k    67.00%
  1331561 requests in 10.10s, 97.78MB read
Requests/sec: 131845.76
Transfer/sec:      9.68MB
[root@localhost gnet-example]# wrk -t 3 -d 10 https://127.0.0.1:443/
Running 10s test @ https://127.0.0.1:443/
  3 threads and 10 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    74.35us  230.93us  18.66ms   99.52%
    Req/Sec    41.21k     4.91k   51.58k    67.33%
  1242107 requests in 10.10s, 91.21MB read
Requests/sec: 122987.05
Transfer/sec:      9.03MB

[root@localhost gnet-example]# wrk --version
wrk 4.1.0 [epoll] Copyright (C) 2012 Will Glozer
[root@localhost gnet-example]# go version
go version go1.22.2 linux/amd64

[kolinfluence]

@leslie-fei my wrk works on the top performing frameworks u can imagine including rust / c / vlang / golang / cpp etc

basically i use the wrk extremely frequently.

hope u can make it work in this version too.

 wrk --version
wrk debian/4.1.0-3build1 [epoll] Copyright (C) 2012 Will Glozer
Usage: wrk <options> <url>                            
  Options:                                            
    -c, --connections <N>  Connections to keep open   
    -d, --duration    <T>  Duration of test           
    -t, --threads     <N>  Number of threads to use   

    -s, --script      <S>  Load Lua script file       
    -H, --header      <H>  Add header to request      
        --latency          Print latency statistics   
        --timeout     <T>  Socket/request timeout     
    -v, --version          Print version details      

  Numeric arguments may include a SI unit (1k, 1M, 1G)
  Time arguments may include a time unit (2s, 2m, 2h)

[leslie-fei]

I don't think it's an issue with wrk, as the version is 4.1.0.

[kolinfluence]

@leslie-fei pls check on your side, i will only run main.go file to check, not binary file.

[leslie-fei]

@kolinfluence There's no issue when testing with wrk on my Linux machine, so I suspect it might be a problem with the executable file. Can you download my compiled package and run the main executable file directly on your Linux machine? Then, test it with wrk to see if it works. gnet-example.zip

[kolinfluence]

@leslie-fei can you try with

GOMAXPROCS=1 ./main?

my wrk is stuck with GOMAXPROCS=1

the problem already mentioned as before, it does not work

[leslie-fei]

@kolinfluence I have tried running with wrk on my machine. Here are go build main.go the results of running GOMAXPROCS=1 ./main. Everything seems to be normal. I don't think it should be a wrk issue. If you believe it's a wrk problem, could you please upload your wrk executable file so that I can try using your version of wrk?

[root@localhost gnet-example]# GOMAXPROCS=1 ./main &
[1] 13015
[root@localhost gnet-example]# [gnet] 2024-04-24T02:04:46.19522445-04:00        INFO    logging/logger.go:256   Launching gnet with 8 event-loops, l                                                  istening on: tcp://:443

[root@localhost gnet-example]# wrk -t 3 -d 10 https://127.0.0.1:443/
Running 10s test @ https://127.0.0.1:443/
  3 threads and 10 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    83.64us  250.27us  17.99ms   99.19%
    Req/Sec    38.42k     5.46k   48.75k    68.54%
  1153909 requests in 10.10s, 84.73MB read
Requests/sec: 114257.14
Transfer/sec:      8.39MB
[root@localhost gnet-example]# wrk -t 3 -d 10 https://127.0.0.1:443/
Running 10s test @ https://127.0.0.1:443/
  3 threads and 10 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    66.40us   71.59us   6.76ms   96.99%
    Req/Sec    43.59k     5.12k   56.81k    73.93%
  1313587 requests in 10.10s, 96.46MB read
Requests/sec: 130057.60
Transfer/sec:      9.55MB
[root@localhost gnet-example]# wrk -t 3 -d 10 https://127.0.0.1:443/
Running 10s test @ https://127.0.0.1:443/
  3 threads and 10 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    73.29us  153.07us   9.07ms   98.37%
    Req/Sec    44.20k     6.30k   55.83k    67.00%
  1331561 requests in 10.10s, 97.78MB read
Requests/sec: 131845.76
Transfer/sec:      9.68MB
[root@localhost gnet-example]# wrk -t 3 -d 10 https://127.0.0.1:443/
Running 10s test @ https://127.0.0.1:443/
  3 threads and 10 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    74.35us  230.93us  18.66ms   99.52%
    Req/Sec    41.21k     4.91k   51.58k    67.33%
  1242107 requests in 10.10s, 91.21MB read
Requests/sec: 122987.05
Transfer/sec:      9.03MB

[kolinfluence]

thank you for trusting me, as attached my wrk version. sry but i'm not running ur version of binary coz i cant trust the binary file. sorry my laptop i need to work with, cant afford to change too many things.

as attached wrk.tar.gz

[kolinfluence]

@leslie-fei can u try ab test too? my ab doesnt seem to work too.

[leslie-fei]

@kolinfluence I roughly know what happened. Let me make some edits later.

[leslie-fei]

@kolinfluence I've put it in the example directory of this repository - https://github.com/leslie-fei/gnettls. It's inconvenient to test with two repositories. Can you clone this project, cd to the example directory, run 'go build main.go', and execute main again?

[kolinfluence]

@leslie-fei i confirm this with you again.

i ran the latest 7a55... commit, i did go get -u

go get -u
go: upgraded github.com/panjf2000/ants/v2 v2.9.0 => v2.9.1
go: upgraded go.uber.org/atomic v1.7.0 => v1.11.0
go: upgraded go.uber.org/multierr v1.6.0 => v1.11.0
go: upgraded go.uber.org/zap v1.21.0 => v1.27.0
go: upgraded golang.org/x/sync v0.6.0 => v0.7.0

GOMAXPROCS=1 ./main

i have issue with your gomaxprocs=1 and i also have issues when after running a few times WITHOUT gomaxprocs=1, it doesnt run wrk anymore and the main program is using ALL cores doing nothing. i have 12 cores.

something's looping inside?

 wrk -d 10 -c 100 https://127.0.0.1:8443/
Running 10s test @ https://127.0.0.1:8443/
  2 threads and 100 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency     0.00us    0.00us   0.00us    -nan%
    Req/Sec     0.00      0.00     0.00      -nan%
  0 requests in 10.02s, 0.00B read
Requests/sec:      0.00
Transfer/sec:       0.00B

[kolinfluence]

@leslie-fei i dunno what's wrong but i'm using amd ryzen ubuntu 22.04

lscpu

Architecture:            x86_64
  CPU op-mode(s):        32-bit, 64-bit
  Address sizes:         48 bits physical, 48 bits virtual
  Byte Order:            Little Endian
CPU(s):                  12
  On-line CPU(s) list:   0-11
Vendor ID:               AuthenticAMD
  Model name:            AMD Ryzen 5 7640HS w/ Radeon 760M Graphics
    CPU family:          25
    Model:               116
    Thread(s) per core:  2
    Core(s) per socket:  6
    Socket(s):           1
    Stepping:            1
    CPU max MHz:         5628.0000
    CPU min MHz:         400.0000
    BogoMIPS:            8584.27
    Flags:               fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clfl
                         ush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm consta
                         nt_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rap
                         l pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes
                          xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a m
                         isalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfct
                         r_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v
                         2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms in
                         vpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb 
                         avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm
                         _occup_llc cqm_mbm_total cqm_mbm_local avx512_bf16 clzero irperf xsaveerptr 
                         rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean fl
                         ushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload vgif x2avic 
                         v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq
                          avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca
                          flush_l1d
Virtualization features: 
  Virtualization:        AMD-V
Caches (sum of all):     
  L1d:                   192 KiB (6 instances)
  L1i:                   192 KiB (6 instances)
  L2:                    6 MiB (6 instances)
  L3:                    16 MiB (1 instance)
NUMA:                    
  NUMA node(s):          1
  NUMA node0 CPU(s):     0-11
Vulnerabilities:         
  Gather data sampling:  Not affected
  Itlb multihit:         Not affected
  L1tf:                  Not affected
  Mds:                   Not affected
  Meltdown:              Not affected
  Mmio stale data:       Not affected
  Retbleed:              Not affected
  Spec rstack overflow:  Vulnerable: Safe RET, no microcode
  Spec store bypass:     Mitigation; Speculative Store Bypass disabled via prctl
  Spectre v1:            Mitigation; usercopy/swapgs barriers and __user pointer sanitization
  Spectre v2:            Vulnerable: eIBRS with unprivileged eBPF
  Srbds:                 Not affected
  Tsx async abort:       Not affected

[leslie-fei]

8443

Could you try curling it to see if it returns the result 'HELLO WORLD' correctly? Could you share the console output after you run the main function?

[root@VM-16-16-centos example]# curl -k https://127.0.0.1:443
Hello world![root@VM-16-16-centos example]#

[kolinfluence]

@leslie-fei i changed it to 8443 so i can run it as a normal user. previous version i ran as root. this latest 7a55 version i ran as 8443. it works sometimes, sometimes dont. i check again.

curl -k https://127.0.0.1:8443
Hello world!

curl -k https://127.0.0.1:8443 2nd run, STUCK.

also main uses 1200% cpu. all of it

[leslie-fei]

@kolinfluence Please pull the latest code, kill the existing process, recompile main.go. I suspect there might be an issue with the infinite loop condition in the handshake. I just added a critical condition. Try it again after starting. First test whether cURL can pass through, and then stress test with wrk if possible.

[kolinfluence]

@leslie-fei first few times of curl show no error, after running a few more curl... 7 or > 8...

previously 2nd run of curl will stuck, now 2nd time works

curl -k https://127.0.0.1:8443
Hello world!
curl -k https://127.0.0.1:8443
Hello world!
curl -k https://127.0.0.1:8443
Hello world!
curl -k https://127.0.0.1:8443
Hello world!

wrk will show a lot more these errors.

 ./main
[gnet] 2024-04-24T06:35:40.347870591-04:00  INFO    logging/logger.go:256   version: 0.0.2
[gnet] 2024-04-24T06:35:40.348201881-04:00  INFO    logging/logger.go:256   Launching gnet with 12 event-loops, listening on: tcp://:8443
[gnet] 2024-04-24T06:35:44.562910937-04:00  ERROR   logging/logger.go:270   max retry handshake inbound buffered: 100
github.com/panjf2000/gnet/v2/pkg/logging.Errorf
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/pkg/logging/logger.go:270
github.com/leslie-fei/gnettls.(*tlsEventHandler).OnTraffic
    /home/ubuntu/gnettls/tls.go:184
github.com/panjf2000/gnet/v2.(*eventloop).read
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/eventloop_unix.go:136
github.com/panjf2000/gnet/v2.(*conn).processIO
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/connection_linux.go:54
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:91
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/internal/netpoll/poller_epoll_default.go:138
github.com/panjf2000/gnet/v2.(*eventloop).run
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:81
golang.org/x/sync/errgroup.(*Group).Go.func1
    /home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78

conclusion : i confirm with you everything works now except that the error message is shown. wrk shows plenty times complete without error. no errors on wrk side. only errors on server side

wrk -d 30 -c 100 https://127.0.0.1:8443/
Running 30s test @ https://127.0.0.1:8443/
  2 threads and 100 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    27.44ms   39.02ms 304.90ms   81.51%
    Req/Sec    79.64k     8.34k  104.38k    81.67%
  4753045 requests in 30.00s, 349.03MB read
Requests/sec: 158424.11
Transfer/sec:     11.63MB

you are very close to solving the issue now, wrk error:

[gnet] 2024-04-24T06:38:25.948253993-04:00  ERROR   logging/logger.go:270   max retry handshake inbound buffered: 62
github.com/panjf2000/gnet/v2/pkg/logging.Errorf
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/pkg/logging/logger.go:270
github.com/leslie-fei/gnettls.(*tlsEventHandler).OnTraffic
    /home/ubuntu/gnettls/tls.go:184
github.com/panjf2000/gnet/v2.(*eventloop).read
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/eventloop_unix.go:136
github.com/panjf2000/gnet/v2.(*conn).processIO
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/connection_linux.go:54
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:91
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/internal/netpoll/poller_epoll_default.go:138
github.com/panjf2000/gnet/v2.(*eventloop).run
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:81
golang.org/x/sync/errgroup.(*Group).Go.func1
    /home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78
[gnet] 2024-04-24T06:38:25.963617821-04:00  ERROR   logging/logger.go:270   max retry handshake inbound buffered: 62
github.com/panjf2000/gnet/v2/pkg/logging.Errorf
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/pkg/logging/logger.go:270
github.com/leslie-fei/gnettls.(*tlsEventHandler).OnTraffic
    /home/ubuntu/gnettls/tls.go:184
github.com/panjf2000/gnet/v2.(*eventloop).read
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/eventloop_unix.go:136
github.com/panjf2000/gnet/v2.(*conn).processIO
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/connection_linux.go:54
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:91
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/internal/netpoll/poller_epoll_default.go:138
github.com/panjf2000/gnet/v2.(*eventloop).run
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:81
golang.org/x/sync/errgroup.(*Group).Go.func1
    /home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78
[gnet] 2024-04-24T06:38:25.99204731-04:00   ERROR   logging/logger.go:270   max retry handshake inbound buffered: 62
github.com/panjf2000/gnet/v2/pkg/logging.Errorf
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/pkg/logging/logger.go:270
github.com/leslie-fei/gnettls.(*tlsEventHandler).OnTraffic
    /home/ubuntu/gnettls/tls.go:184
github.com/panjf2000/gnet/v2.(*eventloop).read
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/eventloop_unix.go:136
github.com/panjf2000/gnet/v2.(*conn).processIO
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/connection_linux.go:54
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:91
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/internal/netpoll/poller_epoll_default.go:138
github.com/panjf2000/gnet/v2.(*eventloop).run
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:81
golang.org/x/sync/errgroup.(*Group).Go.func1
    /home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78

[leslie-fei]

@kolinfluence I will fix it tomorrow

[kolinfluence]

@leslie-fei by the way, it is working toooo fast. the overhead of tls is like nothing. that's extremely impressive (or totally wrong?)

what i mean is, non tls is 165,000 req/s, the older-other tls version, 90,000 req/s.

the one u have is 158,000 req/s.

that's very amazing. i hope it is truly so too. honestly i expect std lib crypto lib to be at least 30% lower than without tls. normally 40% lower is expected.

NEW EDIT : i thought about it for some time, i think it "feels" like all the tls goes through only once? coz this is really too fast. does not seem to make sense. let's see how u fix the "error" first. i guesstimate the best result should be 105k - 120k req/s.

p.s. : thank you for trusting my results. sometimes people think i'm trying to be funny or did the test wrong :D

p.p.s. : i rarely use wechat. pls add me my username kolinfluence. asking u to add me coz i dunno how to add u

[leslie-fei]

@kolinfluence Please pull the latest code and try compiling and running it again. I've already added you on WeChat. What chat application do you usually use? I can also sign up for one.

[leslie-fei]

@kolinfluence If you suspect that TLS might not be effective, you can replace it with a secure certificate. Then update your host domain pointing to your own machine and try accessing it via HTTPS using a browser. I have tested locally that browsers can access HTTPS and verify certificates.

[kolinfluence]

@leslie-fei wrk issues, i changed to port 8443 (just fyi)

latest test with latest main.go

wrk -d 10000 -c 1000 https://127.0.0.1:8443/

mple$ [gnet] 2024-04-24T22:36:08.289166381-04:00    ERROR   logging/logger.go:270   max retry handshake inbound buffered: 62
github.com/panjf2000/gnet/v2/pkg/logging.Errorf
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/pkg/logging/logger.go:270
github.com/leslie-fei/gnettls.(*tlsEventHandler).OnTraffic
    /home/ubuntu/gnettls/tls.go:184
github.com/panjf2000/gnet/v2.(*eventloop).read
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/eventloop_unix.go:136
github.com/panjf2000/gnet/v2.(*conn).processIO
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/connection_linux.go:54
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:91
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/internal/netpoll/poller_epoll_default.go:138
github.com/panjf2000/gnet/v2.(*eventloop).run
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:81
golang.org/x/sync/errgroup.(*Group).Go.func1
    /home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78
[gnet] 2024-04-24T22:36:08.292787816-04:00  ERROR   logging/logger.go:270   max retry handshake inbound buffered: 62
github.com/panjf2000/gnet/v2/pkg/logging.Errorf
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/pkg/logging/logger.go:270
github.com/leslie-fei/gnettls.(*tlsEventHandler).OnTraffic
    /home/ubuntu/gnettls/tls.go:184
github.com/panjf2000/gnet/v2.(*eventloop).read
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/eventloop_unix.go:136
github.com/panjf2000/gnet/v2.(*conn).processIO
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/connection_linux.go:54
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:91
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/internal/netpoll/poller_epoll_default.go:138
github.com/panjf2000/gnet/v2.(*eventloop).run
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:81
golang.org/x/sync/errgroup.(*Group).Go.func1
    /home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78
[gnet] 2024-04-24T22:36:08.294796012-04:00  ERROR   logging/logger.go:270   max retry handshake inbound buffered: 62
github.com/panjf2000/gnet/v2/pkg/logging.Errorf
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/pkg/logging/logger.go:270
github.com/leslie-fei/gnettls.(*tlsEventHandler).OnTraffic
    /home/ubuntu/gnettls/tls.go:184
github.com/panjf2000/gnet/v2.(*eventloop).read
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/eventloop_unix.go:136
github.com/panjf2000/gnet/v2.(*conn).processIO
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/connection_linux.go:54
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:91
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/internal/netpoll/poller_epoll_default.go:138
github.com/panjf2000/gnet/v2.(*eventloop).run
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:81
golang.org/x/sync/errgroup.(*Group).Go.func1
    /home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78
[gnet] 2024-04-24T22:36:08.295296776-04:00  ERROR   logging/logger.go:270   max retry handshake inbound buffered: 62
github.com/panjf2000/gnet/v2/pkg/logging.Errorf
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/pkg/logging/logger.go:270
github.com/leslie-fei/gnettls.(*tlsEventHandler).OnTraffic
    /home/ubuntu/gnettls/tls.go:184
github.com/panjf2000/gnet/v2.(*eventloop).read
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/eventloop_unix.go:136
github.com/panjf2000/gnet/v2.(*conn).processIO
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/connection_linux.go:54
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:91
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/internal/netpoll/poller_epoll_default.go:138
github.com/panjf2000/gnet/v2.(*eventloop).run
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:81
golang.org/x/sync/errgroup.(*Group).Go.func1
    /home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78
[gnet] 2024-04-24T22:36:08.295709887-04:00  ERROR   logging/logger.go:270   max retry handshake inbound buffered: 62
github.com/panjf2000/gnet/v2/pkg/logging.Errorf
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/pkg/logging/logger.go:270
github.com/leslie-fei/gnettls.(*tlsEventHandler).OnTraffic
    /home/ubuntu/gnettls/tls.go:184
github.com/panjf2000/gnet/v2.(*eventloop).read
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/eventloop_unix.go:136
github.com/panjf2000/gnet/v2.(*conn).processIO
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/connection_linux.go:54
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:91
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/internal/netpoll/poller_epoll_default.go:138
github.com/panjf2000/gnet/v2.(*eventloop).run
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:81
golang.org/x/sync/errgroup.(*Group).Go.func1
    /home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78
[gnet] 2024-04-24T22:36:08.300737274-04:00  ERROR   logging/logger.go:270   max retry handshake inbound buffered: 62
github.com/panjf2000/gnet/v2/pkg/logging.Errorf
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/pkg/logging/logger.go:270
github.com/leslie-fei/gnettls.(*tlsEventHandler).OnTraffic
    /home/ubuntu/gnettls/tls.go:184
github.com/panjf2000/gnet/v2.(*eventloop).read
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/eventloop_unix.go:136
github.com/panjf2000/gnet/v2.(*conn).processIO
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/connection_linux.go:54
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:91
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/internal/netpoll/poller_epoll_default.go:138
github.com/panjf2000/gnet/v2.(*eventloop).run
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:81
golang.org/x/sync/errgroup.(*Group).Go.func1
    /home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78
[gnet] 2024-04-24T22:36:08.305137651-04:00  ERROR   logging/logger.go:270   max retry handshake inbound buffered: 62
github.com/panjf2000/gnet/v2/pkg/logging.Errorf
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/pkg/logging/logger.go:270
github.com/leslie-fei/gnettls.(*tlsEventHandler).OnTraffic
    /home/ubuntu/gnettls/tls.go:184
github.com/panjf2000/gnet/v2.(*eventloop).read
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/eventloop_unix.go:136
github.com/panjf2000/gnet/v2.(*conn).processIO
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/connection_linux.go:54
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:91
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/internal/netpoll/poller_epoll_default.go:138
github.com/panjf2000/gnet/v2.(*eventloop).run
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:81
golang.org/x/sync/errgroup.(*Group).Go.func1
    /home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78

[kolinfluence]

@leslie-fei this is fine, no errors wrk -d 10 -c 10 https://127.0.0.1:8443/

Running 10s test @ https://127.0.0.1:8443/
  2 threads and 10 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    37.50us   15.36us   1.39ms   67.71%
    Req/Sec    98.19k    15.10k  113.01k    77.00%
  1953040 requests in 10.00s, 143.42MB read
Requests/sec: 195301.66
Transfer/sec:     14.34MB

this is not fine with errors above wrk -d 10 -c 100 https://127.0.0.1:8443/

Running 10s test @ https://127.0.0.1:8443/
  2 threads and 100 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency   258.09us  450.11us  35.97ms   99.91%
    Req/Sec   103.39k    16.95k  118.09k    76.00%
  2057317 requests in 10.00s, 151.07MB read
Requests/sec: 205666.59
Transfer/sec:     15.10MB

[kolinfluence]

@leslie-fei this has errors generated on server side when used with GOMAXPROCS=1, the whole wrk completes without errors with 150k req/s

wrk -d 10 -c 10 https://127.0.0.1:8443/

[kolinfluence]

@leslie-fei final test for this, conclusion

when ran without errors, it's doing 87k tls req/s per core. (as expected)

BUT

when specifying gomaxprocs parameters will definitely show this error, then the req/s is not accurate

./main
[gnet] 2024-04-24T22:47:34.70975246-04:00   INFO    logging/logger.go:256   version: 0.0.2
[gnet] 2024-04-24T22:47:34.709964455-04:00  INFO    logging/logger.go:256   Launching gnet with 12 event-loops, listening on: tcp://:8443
[gnet] 2024-04-24T22:47:35.92511304-04:00   ERROR   logging/logger.go:270   max retry handshake inbound buffered: 62
github.com/panjf2000/gnet/v2/pkg/logging.Errorf
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/pkg/logging/logger.go:270
github.com/leslie-fei/gnettls.(*tlsEventHandler).OnTraffic
    /home/ubuntu/gnettls/tls.go:184
github.com/panjf2000/gnet/v2.(*eventloop).read
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/eventloop_unix.go:136
github.com/panjf2000/gnet/v2.(*conn).processIO
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/connection_linux.go:54
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:91
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/internal/netpoll/poller_epoll_default.go:138
github.com/panjf2000/gnet/v2.(*eventloop).run
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:81
golang.org/x/sync/errgroup.(*Group).Go.func1
    /home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78
[gnet] 2024-04-24T22:47:35.925228154-04:00  ERROR   logging/logger.go:270   max retry handshake inbound buffered: 62
github.com/panjf2000/gnet/v2/pkg/logging.Errorf
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/pkg/logging/logger.go:270
github.com/leslie-fei/gnettls.(*tlsEventHandler).OnTraffic
    /home/ubuntu/gnettls/tls.go:184
github.com/panjf2000/gnet/v2.(*eventloop).read
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/eventloop_unix.go:136
github.com/panjf2000/gnet/v2.(*conn).processIO
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/connection_linux.go:54
github.com/panjf2000/gnet/v2.(*eventloop).run.func1
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:91
github.com/panjf2000/gnet/v2/internal/netpoll.(*Poller).Polling
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/internal/netpoll/poller_epoll_default.go:138
github.com/panjf2000/gnet/v2.(*eventloop).run
    /home/ubuntu/go/pkg/mod/github.com/panjf2000/gnet/v2@v2.5.0/reactor_epoll_default.go:81
golang.org/x/sync/errgroup.(*Group).Go.func1
    /home/ubuntu/go/pkg/mod/golang.org/x/sync@v0.7.0/errgroup/errgroup.go:78
^C

[leslie-fei]

@kolinfluence please pull latest code build and run again, I may have found the issue. I hope for success. What chat application do you usually use? I can also sign up for one.