[Feature Request] Support SLL handshake alert code (112) when https domain is not found

Describe the feature request

For the https request, frps server will return a 404 Not Found code in HTTP protocol when the domain name is not found.

openssl s_client -debug -servername localhost -connect localhost:443

CONNECTED(00000003)
write to 0x5628c4b96cc0 [0x5628c4baf880] (311 bytes => 311 (0x137))
0000 - 16 03 01 01 32 01 00 01-2e 03 03 32 35 18 ae 08   ....2......25...
0010 - 75 f7 e8 7c e7 bd 7b 6e-d5 56 74 15 9b 4c d1 6a   u..|..{n.Vt..L.j
0020 - f2 f7 35 97 e5 4d 16 a9-da 47 b3 20 b4 4b e7 c3   ..5..M...G. .K..
0030 - e3 c4 b7 45 85 12 88 9d-85 da 72 81 13 dd 99 f5   ...E......r.....
0040 - 50 d7 8b 80 1d 83 47 94-24 38 cb fe 00 3e 13 02   P.....G.$8...>..
0050 - 13 03 13 01 c0 2c c0 30-00 9f cc a9 cc a8 cc aa   .....,.0........
0060 - c0 2b c0 2f 00 9e c0 24-c0 28 00 6b c0 23 c0 27   .+./...$.(.k.#.'
0070 - 00 67 c0 0a c0 14 00 39-c0 09 c0 13 00 33 00 9d   .g.....9.....3..
0080 - 00 9c 00 3d 00 3c 00 35-00 2f 00 ff 01 00 00 a7   ...=.<.5./......
0090 - 00 00 00 0e 00 0c 00 00-09 6c 6f 63 61 6c 68 6f   .........localho
00a0 - 73 74 00 0b 00 04 03 00-01 02 00 0a 00 0c 00 0a   st..............
00b0 - 00 1d 00 17 00 1e 00 19-00 18 00 23 00 00 00 16   ...........#....
00c0 - 00 00 00 17 00 00 00 0d-00 30 00 2e 04 03 05 03   .........0......
00d0 - 06 03 08 07 08 08 08 09-08 0a 08 0b 08 04 08 05   ................
00e0 - 08 06 04 01 05 01 06 01-03 03 02 03 03 01 02 01   ................
00f0 - 03 02 02 02 04 02 05 02-06 02 00 2b 00 09 08 03   ...........+....
0100 - 04 03 03 03 02 03 01 00-2d 00 02 01 01 00 33 00   ........-.....3.
0110 - 26 00 24 00 1d 00 20 fc-45 f9 b4 a2 ee cb aa f9   &.$... .E.......
0120 - bf db 89 8f eb a9 86 95-3a e7 f9 57 74 da 33 a0   ........:..Wt.3.
0130 - c3 a5 e4 68 a3 51 7c                              ...h.Q|
read from 0x5628c4b96cc0 [0x5628c4ba6663] (5 bytes => 5 (0x5))
0000 - 48 54 54 50 2f                                    HTTP/
139671755646272:error:1408F10B:SSL routines:ssl3_get_record:wrong version number:ssl/record/ssl3_record.c:332:
---
no peer certificate available
---
No client certificate CA names sent
---
SSL handshake has read 5 bytes and written 311 bytes
Verification: OK
---
New, (NONE), Cipher is (NONE)
Secure Renegotiation IS NOT supported
Compression: NONE
Expansion: NONE
No ALPN negotiated
Early data was not sent
Verify return code: 0 (ok)
---
read from 0x5628c4b96cc0 [0x5628c4ace180] (8192 bytes => 65 (0x41))
0000 - 31 2e 31 20 34 30 34 20-4e 6f 74 20 46 6f 75 6e   1.1 404 Not Foun
0010 - 64 0d 0a 43 6f 6e 74 65-6e 74 2d 4c 65 6e 67 74   d..Content-Lengt
0020 - 68 3a 20 34 38 39 0d 0a-43 6f 6e 74 65 6e 74 2d   h: 489..Content-
0030 - 54 79 70 65 3a 20 74 65-78 74 2f 68 74 6d 6c 0d   Type: text/html.
0040 - 0a                                                .
read from 0x5628c4b96cc0 [0x5628c4ace180] (8192 bytes => 20 (0x14))
0000 - 53 65 72 76 65 72 3a 20-66 72 70 2f 30 2e 35 31   Server: frp/0.51
0010 - 2e 33 0d 0a                                       .3..
read from 0x5628c4b96cc0 [0x5628c4ace180] (8192 bytes => 2 (0x2))
0000 - 0d 0a                                             ..
read from 0x5628c4b96cc0 [0x5628c4ace180] (8192 bytes => 489 (0x1E9))
0000 - 3c 21 44 4f 43 54 59 50-45 20 68 74 6d 6c 3e 0a   <!DOCTYPE html>.
0010 - 3c 68 74 6d 6c 3e 0a 3c-68 65 61 64 3e 0a 3c 74   <html>.<head>.<t
0020 - 69 74 6c 65 3e 4e 6f 74-20 46 6f 75 6e 64 3c 2f   itle>Not Found</
0030 - 74 69 74 6c 65 3e 0a 3c-73 74 79 6c 65 3e 0a 20   title>.<style>. 
0040 - 20 20 20 62 6f 64 79 20-7b 0a 20 20 20 20 20 20      body {.      
0050 - 20 20 77 69 64 74 68 3a-20 33 35 65 6d 3b 0a 20     width: 35em;. 
0060 - 20 20 20 20 20 20 20 6d-61 72 67 69 6e 3a 20 30          margin: 0
0070 - 20 61 75 74 6f 3b 0a 20-20 20 20 20 20 20 20 66    auto;.        f
0080 - 6f 6e 74 2d 66 61 6d 69-6c 79 3a 20 54 61 68 6f   ont-family: Taho
0090 - 6d 61 2c 20 56 65 72 64-61 6e 61 2c 20 41 72 69   ma, Verdana, Ari
00a0 - 61 6c 2c 20 73 61 6e 73-2d 73 65 72 69 66 3b 0a   al, sans-serif;.
00b0 - 20 20 20 20 7d 0a 3c 2f-73 74 79 6c 65 3e 0a 3c       }.</style>.<
00c0 - 2f 68 65 61 64 3e 0a 3c-62 6f 64 79 3e 0a 3c 68   /head>.<body>.<h
00d0 - 31 3e 54 68 65 20 70 61-67 65 20 79 6f 75 20 72   1>The page you r
00e0 - 65 71 75 65 73 74 65 64-20 77 61 73 20 6e 6f 74   equested was not
00f0 - 20 66 6f 75 6e 64 2e 3c-2f 68 31 3e 0a 3c 70 3e    found.</h1>.<p>
0100 - 53 6f 72 72 79 2c 20 74-68 65 20 70 61 67 65 20   Sorry, the page 
0110 - 79 6f 75 20 61 72 65 20-6c 6f 6f 6b 69 6e 67 20   you are looking 
0120 - 66 6f 72 20 69 73 20 63-75 72 72 65 6e 74 6c 79   for is currently
0130 - 20 75 6e 61 76 61 69 6c-61 62 6c 65 2e 3c 62 72    unavailable.<br
0140 - 2f 3e 0a 50 6c 65 61 73-65 20 74 72 79 20 61 67   />.Please try ag
0150 - 61 69 6e 20 6c 61 74 65-72 2e 3c 2f 70 3e 0a 3c   ain later.</p>.<
0160 - 70 3e 54 68 65 20 73 65-72 76 65 72 20 69 73 20   p>The server is 
0170 - 70 6f 77 65 72 65 64 20-62 79 20 3c 61 20 68 72   powered by <a hr
0180 - 65 66 3d 22 68 74 74 70-73 3a 2f 2f 67 69 74 68   ef="https://gith
0190 - 75 62 2e 63 6f 6d 2f 66-61 74 65 64 69 65 72 2f   ub.com/fatedier/
01a0 - 66 72 70 22 3e 66 72 70-3c 2f 61 3e 2e 3c 2f 70   frp">frp</a>.</p
01b0 - 3e 0a 3c 70 3e 3c 65 6d-3e 46 61 69 74 68 66 75   >.<p><em>Faithfu
01c0 - 6c 6c 79 20 79 6f 75 72-73 2c 20 66 72 70 2e 3c   lly yours, frp.<
01d0 - 2f 65 6d 3e 3c 2f 70 3e-0a 3c 2f 62 6f 64 79 3e   /em></p>.</body>
01e0 - 0a 3c 2f 68 74 6d 6c 3e-0a                        .</html>.
read from 0x5628c4b96cc0 [0x5628c4ace180] (8192 bytes => 0 (0x0))

This violate the https/ssl standard.

Describe alternatives you've considered

A better way for the return would be

CONNECTED(00000003)
write to 0x55ea35975cc0 [0x55ea3598e880] (311 bytes => 311 (0x137))
0000 - 16 03 01 01 32 01 00 01-2e 03 03 e3 b6 71 70 24   ....2........qp$
0010 - d9 fa 7e f8 79 8b a4 8a-71 81 67 ec 86 05 65 a6   ..~.y...q.g...e.
0020 - 6f 5e 7b 20 87 8a 22 66-20 33 ec 20 8e 9a d8 9f   o^{ .."f 3. ....
0030 - 64 05 af 21 7b 5d ff f6-ae 6a 2d 54 30 6d 3f 5a   d..!{]...j-T0m?Z
0040 - b7 45 8f 40 9d 77 cc a7-88 1a 40 78 00 3e 13 02   .E.@.w....@x.>..
0050 - 13 03 13 01 c0 2c c0 30-00 9f cc a9 cc a8 cc aa   .....,.0........
0060 - c0 2b c0 2f 00 9e c0 24-c0 28 00 6b c0 23 c0 27   .+./...$.(.k.#.'
0070 - 00 67 c0 0a c0 14 00 39-c0 09 c0 13 00 33 00 9d   .g.....9.....3..
0080 - 00 9c 00 3d 00 3c 00 35-00 2f 00 ff 01 00 00 a7   ...=.<.5./......
0090 - 00 00 00 0e 00 0c 00 00-09 6c 6f 63 61 6c 68 6f   .........localho
00a0 - 73 74 00 0b 00 04 03 00-01 02 00 0a 00 0c 00 0a   st..............
00b0 - 00 1d 00 17 00 1e 00 19-00 18 00 23 00 00 00 16   ...........#....
00c0 - 00 00 00 17 00 00 00 0d-00 30 00 2e 04 03 05 03   .........0......
00d0 - 06 03 08 07 08 08 08 09-08 0a 08 0b 08 04 08 05   ................
00e0 - 08 06 04 01 05 01 06 01-03 03 02 03 03 01 02 01   ................
00f0 - 03 02 02 02 04 02 05 02-06 02 00 2b 00 09 08 03   ...........+....
0100 - 04 03 03 03 02 03 01 00-2d 00 02 01 01 00 33 00   ........-.....3.
0110 - 26 00 24 00 1d 00 20 4b-ee 04 97 84 93 e4 b4 5c   &.$... K.......\
0120 - 0a 4a cb 84 45 31 e6 72-48 d7 93 17 84 43 91 75   .J..E1.rH....C.u
0130 - ce 4a 8d 35 c2 d0 5a                              .J.5..Z
read from 0x55ea35975cc0 [0x55ea35985663] (5 bytes => 5 (0x5))
0000 - 15 03 03 00 02                                    .....
read from 0x55ea35975cc0 [0x55ea35985668] (2 bytes => 2 (0x2))
0000 - 02 70                                             .p
140105365730624:error:14094458:SSL routines:ssl3_read_bytes:tlsv1 unrecognized name:ssl/record/rec_layer_s3.c:1544:SSL alert number 112
---
no peer certificate available
---
No client certificate CA names sent
---
SSL handshake has read 7 bytes and written 311 bytes
Verification: OK
---
New, (NONE), Cipher is (NONE)
Secure Renegotiation IS NOT supported
Compression: NONE
Expansion: NONE
No ALPN negotiated
Early data was not sent
Verify return code: 0 (ok)
---
read from 0x55ea35975cc0 [0x55ea358ad180] (8192 bytes => 0 (0x0))

According to RFC4366, a SSL alert number 112 indicates unrecognized_name.

"unrecognized_name": this alert is sent by servers that receive a server_name extension request, but do not recognize the server name. This message MAY be fatal.

This matches our case exactly.

In Chrome it will show something like this. Screenshot from 2023-09-17 00-38-27

I attach a patch here to address this issue. I can open a pull request for this patch if it sounds good to you. Thank you!

diff --git a/pkg/util/tcpmux/httpconnect.go b/pkg/util/tcpmux/httpconnect.go
index 650891f..e820f22 100644
--- a/pkg/util/tcpmux/httpconnect.go
+++ b/pkg/util/tcpmux/httpconnect.go
@@ -38,7 +38,7 @@ type HTTPConnectTCPMuxer struct {

 func NewHTTPConnectTCPMuxer(listener net.Listener, passthrough bool, timeout time.Duration) (*HTTPConnectTCPMuxer, error) {
    ret := &HTTPConnectTCPMuxer{passthrough: passthrough}
-   mux, err := vhost.NewMuxer(listener, ret.getHostFromHTTPConnect, timeout)
+   mux, err := vhost.NewMuxer(listener, ret.getHostFromHTTPConnect, vhostFailed, timeout)
    mux.SetCheckAuthFunc(ret.auth).
        SetSuccessHookFunc(ret.sendConnectResponse)
    ret.Muxer = mux
@@ -92,6 +92,16 @@ func (muxer *HTTPConnectTCPMuxer) auth(c net.Conn, username, password string, re
    return false, nil
 }

+func vhostFailed(c net.Conn) {
+   res := vhost.NotFoundResponse()
+   if res.Body != nil {
+       defer res.Body.Close()
+   }
+   _ = res.Write(c)
+   _ = c.Close()
+   return
+}
+
 func (muxer *HTTPConnectTCPMuxer) getHostFromHTTPConnect(c net.Conn) (net.Conn, map[string]string, error) {
    reqInfoMap := make(map[string]string, 0)
    sc, rd := libnet.NewSharedConn(c)
diff --git a/pkg/util/vhost/http.go b/pkg/util/vhost/http.go
index af3a4ab..7b914ce 100644
--- a/pkg/util/vhost/http.go
+++ b/pkg/util/vhost/http.go
@@ -251,7 +251,7 @@ func (rp *HTTPReverseProxy) connectHandler(rw http.ResponseWriter, req *http.Req

    remote, err := rp.CreateConnection(req.Context().Value(RouteInfoKey).(*RequestRouteInfo), false)
    if err != nil {
-       _ = notFoundResponse().Write(client)
+       _ = NotFoundResponse().Write(client)
        client.Close()
        return
    }
diff --git a/pkg/util/vhost/https.go b/pkg/util/vhost/https.go
index e15c190..f6b5187 100644
--- a/pkg/util/vhost/https.go
+++ b/pkg/util/vhost/https.go
@@ -28,7 +28,7 @@ type HTTPSMuxer struct {
 }

 func NewHTTPSMuxer(listener net.Listener, timeout time.Duration) (*HTTPSMuxer, error) {
-   mux, err := NewMuxer(listener, GetHTTPSHostname, timeout)
+   mux, err := NewMuxer(listener, GetHTTPSHostname, vhostFailed, timeout)
    if err != nil {
        return nil, err
    }
@@ -45,6 +45,7 @@ func GetHTTPSHostname(c net.Conn) (_ net.Conn, _ map[string]string, err error) {
    }

    reqInfoMap["Host"] = clientHello.ServerName
+
    reqInfoMap["Scheme"] = "https"
    return sc, reqInfoMap, nil
 }
@@ -69,6 +70,12 @@ func readClientHello(reader io.Reader) (*tls.ClientHelloInfo, error) {
    return hello, nil
 }

+func vhostFailed(c net.Conn) {
+   // Alert with alertUnrecognizedName
+   tls.Server(c, &tls.Config{}).Handshake()
+   c.Close()
+}
+
 type readOnlyConn struct {
    reader io.Reader
 }
diff --git a/pkg/util/vhost/resource.go b/pkg/util/vhost/resource.go
index e09edf2..d78082b 100644
--- a/pkg/util/vhost/resource.go
+++ b/pkg/util/vhost/resource.go
@@ -67,7 +67,7 @@ func getNotFoundPageContent() []byte {
    return buf
 }

-func notFoundResponse() *http.Response {
+func NotFoundResponse() *http.Response {
    header := make(http.Header)
    header.Set("server", "frp/"+version.Full())
    header.Set("Content-Type", "text/html")
diff --git a/pkg/util/vhost/vhost.go b/pkg/util/vhost/vhost.go
index 6051a21..f29b9e4 100644
--- a/pkg/util/vhost/vhost.go
+++ b/pkg/util/vhost/vhost.go
@@ -46,6 +46,7 @@ type (
    authFunc        func(conn net.Conn, username, password string, reqInfoMap map[string]string) (bool, error)
    hostRewriteFunc func(net.Conn, string) (net.Conn, error)
    successHookFunc func(net.Conn, map[string]string) error
+   failFunc        func(net.Conn)
 )

 // Muxer is a functional component used for https and tcpmux proxies.
@@ -58,6 +59,7 @@ type Muxer struct {
    vhostFunc      muxFunc
    checkAuth      authFunc
    successHook    successHookFunc
+   failFunc       failFunc
    rewriteHost    hostRewriteFunc
    registryRouter *Routers
 }
@@ -65,12 +67,14 @@ type Muxer struct {
 func NewMuxer(
    listener net.Listener,
    vhostFunc muxFunc,
+   failFunc failFunc,
    timeout time.Duration,
 ) (mux *Muxer, err error) {
    mux = &Muxer{
        listener:       listener,
        timeout:        timeout,
        vhostFunc:      vhostFunc,
+       failFunc:       failFunc,
        registryRouter: NewRouters(),
    }
    go mux.run()
@@ -206,13 +210,8 @@ func (v *Muxer) handle(c net.Conn) {
    httpUser := reqInfoMap["HTTPUser"]
    l, ok := v.getListener(name, path, httpUser)
    if !ok {
-       res := notFoundResponse()
-       if res.Body != nil {
-           defer res.Body.Close()
-       }
-       _ = res.Write(c)
        log.Debug("http request for host [%s] path [%s] httpUser [%s] not found", name, path, httpUser)
-       _ = c.Close()
+       v.failFunc(sConn)
        return
    }

Affected area

[ ] Docs
[ ] Installation
[x] Performance and Scalability
[ ] Security
[X] User Experience
[ ] Test and Release
[ ] Developer Infrastructure
[ ] Client Plugin
[ ] Server Plugin
[ ] Extensions
[ ] Others

fatedier / frp

[Feature Request] Support SLL handshake alert code (112) when https domain is not found #3618

Describe the feature request

Describe alternatives you've considered

Affected area