InsecureSkipVerify=false的时候，无法进行TLS连接

[ericjing83]

大佬您好， 我在测试nbio的Tls websocket server和gorilla client的连接，测了好几次，都出现了EOF的错误.服务器端和客户端都在同一台物理机上。我仅仅开启了常规的单向TLS握手（仅仅客户端会验证服务器端的真假） 奇怪的是，当我使用一模一样的证书，在同一台物理服务器上，让gorilla client（完全一模一样的代码）连接gorilla fasthttp的websocket server(同样开启TLS)，连接就会成功（而且我开启了双向TLS握手）。我确定所有的服务器和客户端证书和key都没问题。（证书没有啥特别的，就是测试用的RSA的证书） 想让您帮忙看看EOF错误的原因。 以下是nbio TLS服务端的代码：

package main import ( "context" "crypto/x509" "flag" "fmt" "github.com/lesismal/llib/std/crypto/tls" "github.com/lesismal/nbio/nbhttp" "io/ioutil" "log" "net/http" "os" "os/signal" "time" "github.com/lesismal/nbio/nbhttp/websocket" ) var ( addr = flag.String("addr", "0.0.0.0:8887", "http service address") hub = Hub{ broadcast: make(chan []byte), register: make(chan *websocket.Conn), unregister: make(chan *websocket.Conn), clients: make(map[*websocket.Conn]bool), } upgrader = newUpgrader() ) func newUpgrader() *websocket.Upgrader { u := websocket.NewUpgrader() u.KeepaliveTime = 60 * time.Second u.OnMessage(func(c *websocket.Conn, messageType websocket.MessageType, message []byte) { hub.broadcast <- message }) u.OnClose(func(c *websocket.Conn, err error) { hub.unregister <- c }) return u } func serveWs(w http.ResponseWriter, r *http.Request) { fmt.Println("hi calling Upgrade") conn, err := upgrader.Upgrade(w, r, nil) if err != nil { log.Println(err) return } fmt.Println("Upgraded:", conn.RemoteAddr().String()) hub.register <- conn } // Hub maintains the set of active clients and broadcasts messages to the clients. type Hub struct { // Registered clients. clients map[*websocket.Conn]bool // Inbound messages from the clients. broadcast chan []byte // Register requests from the clients. register chan *websocket.Conn // Unregister requests from clients. unregister chan *websocket.Conn } func (h *Hub) run() { for { select { case client := <-h.register: h.clients[client] = true case client := <-h.unregister: if _, ok := h.clients[client]; ok { delete(h.clients, client) } case message := <-h.broadcast: for conn := range h.clients { conn.WriteMessage(websocket.TextMessage, message) } } } } func addTrust(pool *x509.CertPool, path string) { aCrt, err := ioutil.ReadFile(path) if err != nil { fmt.Println("ReadFile err:", err) return } pool.AppendCertsFromPEM(aCrt) } func main() { fmt.Println("hi,greeting from mws!") flag.Parse() go hub.run() var rsaCertPEM = []byte(`-----BEGIN CERTIFICATE----- -----END CERTIFICATE-----`) var rsaKeyPEM = []byte(`-----BEGIN RSA PRIVATE KEY----- -----END RSA PRIVATE KEY-----`) cert, err := tls.X509KeyPair(rsaCertPEM, rsaKeyPEM) if err != nil { fmt.Println("tls.X509KeyPair failed: ", err) } cipherSuites := []uint16{ tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, } //pool := x509.NewCertPool() //测试里暂时没开启，仅仅在服务器端验证客户端的真假的时候需要 //addTrust(pool, "/usr/local/certificate/ca.crt") //测试里暂时没开启，仅仅在服务器端验证客户端的真假的时候需要 //addTrust(pool, "/usr/local/certificate/client_testcloud.crt") //测试里暂时没开启，仅仅在服务器端验证客户端的真假的时候需要 tlsConfig := &tls.Config{ Certificates: []tls.Certificate{cert}, CipherSuites: cipherSuites, //InsecureSkipVerify: true, } mux := &http.ServeMux{} mux.HandleFunc("/", serveWs) engine := nbhttp.NewEngine(nbhttp.Config{ Network: "tcp", Addrs: []string{*addr}, TLSConfig: tlsConfig, MaxLoad: 1000000, ReleaseWebsocketPayload: true, Handler: mux, MessageHandlerPoolSize: 10000, IOMod: nbhttp.IOModNonBlocking, // websocket 单帧最大载荷 MaxWebsocketFramePayloadSize: 1024 * 16, //this is a recommend value from chrome }) err = engine.Start() if err != nil { fmt.Printf("nbio.Start failed: %v\n", err) return } interrupt := make(chan os.Signal, 1) signal.Notify(interrupt, os.Interrupt) <-interrupt ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) defer cancel() engine.Shutdown(ctx) } 以下是gorilla websocket客户端的代码，这个代码进一步调用了封装了的gowsclient（这个库在生产环境我用了很久了，肯定没有问题）: package main import ( "fmt" "github.com/luaxlou/gowsclient" ) var remoteserverURL_mws = "www.testcloud.com:8887" var gorillaclient_mws *gowsclient.Client func main() { fmt.Println("try to connect to mws as a gorilla client") var err error gorillaclient_mws, err = gowsclient.New(remoteserverURL_mws) if err != nil { fmt.Println("gorilla client mws connection has error: ", err) return } gorillaclient_mws.OnConnect = func() { fmt.Println("OnConnect for gorilla client mws") } go gorillaclient_mws.Connect(0, "https://wsserver") ch := make(chan bool, 1) <-ch } 以下是gowsclient的代码: package gowsclient import ( "crypto/tls" "crypto/x509" "fmt" "github.com/gorilla/websocket" "io/ioutil" "log" "net/http" "net/url" "strconv" "time" "unsafe" ) const ( pongWait = 60 * time.Second pingPeriod = (pongWait * 9) / 10 ) var headervalue = make([]byte,0,100) var wsclientuserid int var ws_origin string var isautoconnect = true type Message struct { Mt int Message []byte } type Client struct { conn *websocket.Conn wsUrl *url.URL header *http.Header sendCh chan *Message OnConnect func() OnReceive func(msg *Message) } func New(rawUrl string) (*Client, error) { return NewWithHeader(rawUrl, nil) } func (c *Client)SetUrl(rawUrl string) { wsUrl, err := url.Parse(rawUrl) if err != nil { return } c.wsUrl = wsUrl } func NewWithHeader(rawUrl string, header *http.Header) (*Client, error) { wsUrl, err := url.Parse(rawUrl) if err != nil { return nil, err } c := &Client{ header: header, wsUrl: wsUrl, sendCh: make(chan *Message), } fmt.Println("check 1 c.wsUrl=",c.wsUrl.String()) return c, nil } func (c *Client) reconnect() { fmt.Println("try to reconnect") if isautoconnect { c.Close() c.Connect(wsclientuserid,ws_origin) } } func (c *Client) Close() { if c.conn != nil { c.conn.Close() } } func (c *Client) Close_NoReconnect() { if c.conn != nil { isautoconnect=false fmt.Println("isautoconnect has been set to false") c.conn.WriteMessage(websocket.TextMessage, []byte{'c'}) c.conn.Close() } } func addTrust(pool*x509.CertPool, path string) { aCrt, err := ioutil.ReadFile(path) if err!= nil { fmt.Println("ReadFile err:",err) return } pool.AppendCertsFromPEM(aCrt) } func Str2Bytes(s string) []byte { x := (*[2]uintptr)(unsafe.Pointer(&s)) h := [3]uintptr{x[0],x[1],x[1]} return *(*[]byte)(unsafe.Pointer(&h)) } func (c *Client) Connect(userid int, origin string) { wsclientuserid=userid ws_origin=origin var header http.Header if c.header != nil { header = *c.header }else { fmt.Println("nil header") } header = make(map[string][]string) header.Add("Origin",origin) headervalue = headervalue[:0] headervalue=append(headervalue, Str2Bytes(strconv.Itoa(userid))...) headervalue=append(headervalue, []byte{';','0',';','0',';','1'}...) header.Add("Sec-WebSocket-Protocol",*(*string)(unsafe.Pointer(&headervalue))) //added start pool := x509.NewCertPool() addTrust(pool,"/usr/local/certificate/ca.crt") cliCrt, err := tls.LoadX509KeyPair("/usr/local/certificate/client_testcloud.crt", "/usr/local/certificate/client_testcloud.key") //only be used when server(mws) check whether the client is a fake one //todo, use tls.X509KeyPair() as embeded to replace tls.LoadX509KeyPair tlsConfig := tls.Config{ //InsecureSkipVerify: true, RootCAs: pool, Certificates: []tls.Certificate{cliCrt}, } dial := websocket.Dialer{TLSClientConfig: &tlsConfig} u := url.URL{Scheme: "wss", Host: c.wsUrl.String(), Path: "/"} ws, _, err := dial.Dial(u.String(), header) if err != nil { fmt.Println("xxxdsdsd ",err) log.Println(err) } fmt.Println("check 2, serverurl=",u.String()) //added end //ws, _, err := websocket.DefaultDialer.Dial(c.wsUrl.String(), header) c.conn = ws if err == nil { fmt.Println("Dial: connection was successfully established with %s\n", c.wsUrl.String()) c.start() if c.OnConnect != nil { c.OnConnect() } return } log.Println("Could not connect to Server:", err.Error()) after := time.After(time.Second * 2) <-after if isautoconnect { fmt.Println("do auto connect") c.Connect(wsclientuserid,ws_origin) } } func (c *Client) WriteMessage(msg *Message) { if c.sendCh != nil { c.sendCh <- msg } } func (c *Client) handleWrite() { ticker := time.NewTicker(pingPeriod) defer func() { ticker.Stop() fmt.Println("gorilla client writer is stopped now") }() for { select { case msg := <-c.sendCh: if msg.Mt==websocket.TextMessage && len(msg.Message)==1 && msg.Message[0]=='c' { return } if c.conn == nil { return } err := c.conn.WriteMessage(msg.Mt, msg.Message) fmt.Println("monitor write action") if err != nil { return } case <-ticker.C: if c.conn == nil { return } if err := c.conn.WriteMessage(websocket.PingMessage, []byte{}); err != nil { return } } } } func (c *Client) start() { go c.loopRead() go c.handleWrite() } func (c *Client) loopRead() { c.conn.SetPongHandler(func(string) error { c.conn.SetReadDeadline(time.Now().Add(pongWait)) return nil }) defer c.reconnect() for { mt, message, err := c.conn.ReadMessage() if err != nil { log.Println("Read error :", err.Error()) break } if c.OnReceive != nil { c.OnReceive(&Message{ Mt: mt, Message: message, }) } } } 跪求大佬看看哪里有问题，我试了一上午，没有成功

[lesismal]

TLS 用 AddrsTLS 再试试, 如果还跑不起来, 请把你的代码整理下markdown或者直接上传压缩包给我一份直接能跑起来的我试试:

engine := nbhttp.NewEngine(nbhttp.Config{
    Network:                 "tcp",
    AddrsTLS:                []string{*addr},
    TLSConfig:               tlsConfig,
    MaxLoad:                 1000000,
    ReleaseWebsocketPayload: true,
    Handler:                 mux,
    MessageHandlerPoolSize:  10000,
    IOMod:                   nbhttp.IOModNonBlocking,
    // websocket 单帧最大载荷
    MaxWebsocketFramePayloadSize: 1024 * 16, //this is a recommend value from chrome
})

[ericjing83]

大佬，我用重试了下，现在可以了。双向TLS握手也可以通过。我增加了自定义origin check的逻辑。 非常感谢指点。我贡献下测试代码和测试用的证书，在附件里。 temp.zip

[lesismal]

好的, 感谢反馈, 欢迎使用!

[ericjing83]

大佬， 顺便问下，我上面的代码，如果按照这个链接：https://github.com/lesismal/nbio/issues/275，仅仅把mux的代码替换为iris框架，真正处理websocket收发消息还是基于nbio，这样，在建立连接的时候，性能会提升吗？

[ericjing83]

我不是很确定，这样替换成了iris之后，在建立连接的时候，TLS握手是不是还会用大佬编写的异步TLS的算法？我希望用nbio自己的异步的TLS算法来处理握手。如果iris和这个问题冲突了，那我还是切换回mux。

[lesismal]

我不是很确定，这样替换成了iris之后，在建立连接的时候，TLS握手是不是还会用大佬编写的异步TLS的算法？我希望用nbio自己的异步的TLS算法来处理握手。如果iris和这个问题冲突了，那我还是切换回mux。

iris以及其他基于标准库的框架主要是用于路由功能, 路由本身与下面传输层使用什么进行数据传输的耦合不大, 除非特殊需要和一些nbio异步无法直接兼容的功能, 其他基本没影响.

[lesismal]

顺便问下，我上面的代码，如果按照这个链接：https://github.com/lesismal/nbio/issues/275，仅仅把mux的代码替换为iris框架，真正处理websocket收发消息还是基于nbio，这样，在建立连接的时候，性能会提升吗？

建立连接不会有性能提升, nbio的websocket功能有点复杂, ws建立连接和握手可能比其他框架略慢, 但这块性能差别不大除非极端性能要求否则基本可以忽略. 如果不是海量连接, nbio poller数据收发的性能也不如标准库方案, 优势主要是海量连接的内存gc和稳定性