hhxsv5 / laravel-s

LaravelS is an out-of-the-box adapter between Laravel/Lumen and Swoole.
MIT License
3.83k stars 470 forks source link

Custom process stop responding after 4minutes #271

Open juslintek opened 4 years ago

juslintek commented 4 years ago
  1. Your software version (Screenshot of your startup)

    Software Version
    PHP 7.4.3
    Swoole 4.5.2
    Laravel 6.18.20
    Ubuntu 18.04
    Supervisord 3.3.1
    Laravel-s 3.7.3
  2. Detail description about this issue(error/log)

[2020-06-24 09:14:20 *7132.0]   WARNING swSignalfd_onSignal (ERRNO 707): Unable to find callback function for signal Broken pipe: 13
[2020-06-24 09:14:20 *7135.3]   WARNING swSignalfd_onSignal (ERRNO 707): Unable to find callback function for signal Broken pipe: 13
[2020-06-24 09:16:00 *7132.0]   NOTICE  swFactoryProcess_finish (ERRNO 1004): send 379 byte failed, because connection[fd=71] is closed
[2020-06-24 09:16:14 *7133.1]   NOTICE  swFactoryProcess_finish (ERRNO 1004): send 439 byte failed, because connection[fd=72] is closed
[2020-06-24 09:16:14 *7134.2]   NOTICE  swFactoryProcess_finish (ERRNO 1004): send 497 byte failed, because connection[fd=73] is closed
[2020-06-25 05:57:58 *7132.0]   WARNING swSignalfd_onSignal (ERRNO 707): Unable to find callback function for signal Broken pipe: 13
[2020-06-25 05:57:58 *7135.3]   WARNING swSignalfd_onSignal (ERRNO 707): Unable to find callback function for signal Broken pipe: 13
[2020-06-25 09:22:21 #7104.4]   INFO    Server is shutdown now
[2020-06-25 09:27:50 #109996.4] INFO    Server is shutdown now
[2020-06-25 09:29:12 #110749.4] INFO    Server is shutdown now
[2020-06-25 09:31:06 #111021.4] INFO    Server is shutdown now
[2020-06-25 09:55:33 #111413.4] INFO    Server is shutdown now
  1. Some reproducible code blocks and steps

Steps to reproduce the same as in that issue. https://github.com/hhxsv5/laravel-s/issues/258

Just changed heartbeat in config these:

        'heartbeat_idle_time' => 30,
        'heartbeat_check_interval' => 10,

And using sock instead of tcp. define('SWOOLE_SOCK_UNIX_STREAM', 5);

Nginx config:

nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
# configuration file /etc/nginx/nginx.conf:
user user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;

events {
    worker_connections 768;
    # multi_accept on;
}

http {

    ##
    # Basic Settings
    ##

    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    # server_tokens off;

    # server_names_hash_bucket_size 64;
    # server_name_in_redirect off;

    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    ##
    # SSL Settings
    ##

    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
    ssl_prefer_server_ciphers on;

    ##
    # Logging Settings
    ##

    access_log /var/log/nginx/access.log;
    error_log /var/log/nginx/error.log;

    ##
    # Gzip Settings
    ##

    gzip on;

    # gzip_vary on;
    # gzip_proxied any;
    # gzip_comp_level 6;
    # gzip_buffers 16 8k;
    # gzip_http_version 1.1;
    # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;

    ##
    # Virtual Host Configs
    ##

    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}

# configuration file /etc/nginx/modules-enabled/50-mod-http-auth-pam.conf:
load_module modules/ngx_http_auth_pam_module.so;

# configuration file /etc/nginx/modules-enabled/50-mod-http-dav-ext.conf:
load_module modules/ngx_http_dav_ext_module.so;

# configuration file /etc/nginx/modules-enabled/50-mod-http-echo.conf:
load_module modules/ngx_http_echo_module.so;

# configuration file /etc/nginx/modules-enabled/50-mod-http-geoip.conf:
load_module modules/ngx_http_geoip_module.so;

# configuration file /etc/nginx/modules-enabled/50-mod-http-geoip2.conf:
load_module modules/ngx_http_geoip2_module.so;

# configuration file /etc/nginx/modules-enabled/50-mod-http-image-filter.conf:
load_module modules/ngx_http_image_filter_module.so;

# configuration file /etc/nginx/modules-enabled/50-mod-http-subs-filter.conf:
load_module modules/ngx_http_subs_filter_module.so;

# configuration file /etc/nginx/modules-enabled/50-mod-http-upstream-fair.conf:
load_module modules/ngx_http_upstream_fair_module.so;

# configuration file /etc/nginx/modules-enabled/50-mod-http-xslt-filter.conf:
load_module modules/ngx_http_xslt_filter_module.so;

# configuration file /etc/nginx/modules-enabled/50-mod-mail.conf:
load_module modules/ngx_mail_module.so;

# configuration file /etc/nginx/modules-enabled/50-mod-ssl-ct.conf:
load_module modules/ngx_ssl_ct_module.so;
load_module modules/ngx_http_ssl_ct_module.so;

# configuration file /etc/nginx/modules-enabled/50-mod-stream.conf:
load_module modules/ngx_stream_module.so;

# configuration file /etc/nginx/mime.types:

types {
    text/html                             html htm shtml;
    text/css                              css;
    text/xml                              xml;
    image/gif                             gif;
    image/jpeg                            jpeg jpg;
    application/javascript                js;
    application/atom+xml                  atom;
    application/rss+xml                   rss;

    text/mathml                           mml;
    text/plain                            txt;
    text/vnd.sun.j2me.app-descriptor      jad;
    text/vnd.wap.wml                      wml;
    text/x-component                      htc;

    image/png                             png;
    image/tiff                            tif tiff;
    image/vnd.wap.wbmp                    wbmp;
    image/x-icon                          ico;
    image/x-jng                           jng;
    image/x-ms-bmp                        bmp;
    image/svg+xml                         svg svgz;
    image/webp                            webp;

    application/font-woff                 woff;
    application/java-archive              jar war ear;
    application/json                      json;
    application/mac-binhex40              hqx;
    application/msword                    doc;
    application/pdf                       pdf;
    application/postscript                ps eps ai;
    application/rtf                       rtf;
    application/vnd.apple.mpegurl         m3u8;
    application/vnd.ms-excel              xls;
    application/vnd.ms-fontobject         eot;
    application/vnd.ms-powerpoint         ppt;
    application/vnd.wap.wmlc              wmlc;
    application/vnd.google-earth.kml+xml  kml;
    application/vnd.google-earth.kmz      kmz;
    application/x-7z-compressed           7z;
    application/x-cocoa                   cco;
    application/x-java-archive-diff       jardiff;
    application/x-java-jnlp-file          jnlp;
    application/x-makeself                run;
    application/x-perl                    pl pm;
    application/x-pilot                   prc pdb;
    application/x-rar-compressed          rar;
    application/x-redhat-package-manager  rpm;
    application/x-sea                     sea;
    application/x-shockwave-flash         swf;
    application/x-stuffit                 sit;
    application/x-tcl                     tcl tk;
    application/x-x509-ca-cert            der pem crt;
    application/x-xpinstall               xpi;
    application/xhtml+xml                 xhtml;
    application/xspf+xml                  xspf;
    application/zip                       zip;

    application/octet-stream              bin exe dll;
    application/octet-stream              deb;
    application/octet-stream              dmg;
    application/octet-stream              iso img;
    application/octet-stream              msi msp msm;

    application/vnd.openxmlformats-officedocument.wordprocessingml.document    docx;
    application/vnd.openxmlformats-officedocument.spreadsheetml.sheet          xlsx;
    application/vnd.openxmlformats-officedocument.presentationml.presentation  pptx;

    audio/midi                            mid midi kar;
    audio/mpeg                            mp3;
    audio/ogg                             ogg;
    audio/x-m4a                           m4a;
    audio/x-realaudio                     ra;

    video/3gpp                            3gpp 3gp;
    video/mp2t                            ts;
    video/mp4                             mp4;
    video/mpeg                            mpeg mpg;
    video/quicktime                       mov;
    video/webm                            webm;
    video/x-flv                           flv;
    video/x-m4v                           m4v;
    video/x-mng                           mng;
    video/x-ms-asf                        asx asf;
    video/x-ms-wmv                        wmv;
    video/x-msvideo                       avi;
}

# Default server configuration
#
server {
    listen 80 default_server;
    listen [::]:80 default_server;

    root /var/www/html;

    # Add index.php to the list if you are using PHP
    index index.html index.htm index.nginx-debian.html;

    server_name _;

    location / {
        # First attempt to serve request as file, then
        # as directory, then fall back to displaying a 404.
        try_files $uri $uri/ =404;
    }
}

map $http_upgrade $connection_upgrade {
    default upgrade;
    ''      close;
}

upstream swoole {
    server unix:/home/user/swoole.sock weight=5 max_fails=3 fail_timeout=30s;
    keepalive 16;
}

server {
    listen 80;
        listen 443 http2 ssl;
        #listen 80;

        server_name test.test;

        ssl_certificate      /etc/letsencrypt/live/test.test/fullchain.pem;
        ssl_certificate_key  /etc/letsencrypt/live/test.test/privkey.pem;

        root /home/myg2/myg2/public;
        index index.php;

    location /socket.io {
        proxy_pass http://localhost:6001; #could be localhost if Echo and NginX are on the same box
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "Upgrade";
    }

    location / {
                try_files $uri @laravels;
    }

    location ~* \.php$ {
                return 404;
        }

        location @laravels {
               proxy_http_version 1.1;
               proxy_connect_timeout 60s;
               proxy_send_timeout 60s;
               proxy_read_timeout 60s;
               proxy_set_header Connection "";
               proxy_set_header X-Real-IP $remote_addr;
               proxy_set_header X-Real-PORT $remote_port;
               proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
               proxy_set_header Host $http_host;
               proxy_set_header Scheme $scheme;
               proxy_set_header Server-Protocol $server_protocol;
               proxy_set_header Server-Name $server_name;
               proxy_set_header Server-Addr $server_addr;
               proxy_set_header Server-Port $server_port;
               proxy_pass http://swoole;
        }
}

Supervisor config:

cat supervisord.conf
; supervisor config file

[unix_http_server]
file=/var/run/supervisor.sock   ; (the path to the socket file)
chmod=0700                       ; sockef file mode (default 0700)

[supervisord]
logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
childlogdir=/var/log/supervisor            ; ('AUTO' child log dir, default $TEMP)

loglevel=debug
logfile_backups=5
user=root

; the below section must remain in the config file for RPC
; (supervisorctl/web interface) to work, additional interfaces may be
; added by defining them in separate rpcinterface: sections
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

[supervisorctl]
serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL  for a unix socket

; The [include] section can just contain the "files" setting.  This
; setting can list multiple files (separated by whitespace or
; newlines).  It can also contain wildcards.  The filenames are
; interpreted as relative to this file.  Included files *cannot*
; include files themselves.

[program:horizon]
process_name=%(program_name)s
command=php /home/user/myg2/artisan horizon
autostart=true
autorestart=true
user=user
stdout_logfile=/home/user/logs/horizon.log
stdout_logfile_maxbytes=10485760
stderr_logfile=/home/user/logs/horizon_error.log
stderr_logfile_maxbytes=10485760
stopwaitsecs=3600

[program:swoole]
process_name=%(program_name)s
command=/home/user/project/bin/laravels start -i
numprocs=1
autostart=true
autorestart=true
startretries=3
umask=122
user=user
stdout_logfile=/home/user/logs/laravels.log
stdout_logfile_maxbytes=10485760
stderr_logfile=/home/user/logs/laravels_error.log
stderr_logfile_maxbytes=10485760

P.S. This doesn't happen on Docker. This happens only in Azure VPS.

juslintek commented 4 years ago

It seems, that you have to terminate connection every 20 seconds and exit process, then it works. Then the process is instantiated and it's good. But it is a kind of nasty solution. I still need to experiment with timeouts. At the moment timeout is 20 on $frame = $client->recv(20);. After I execute:

class TestProcess implements CustomProcessInterface
{
    private static bool $quit = false;

    public static function callback(Server $swoole, Process $process): void
    {

       $client = new Client(
            'echo.websocket.org',
            443
        );

        $client->set([
            'reconnect' => 3,
            'timeout' => 30,
            'defer' => true,
        ]);

        $client->setHeaders([
            'Host' => 'echo.websocket.org',
            'User-Agent' => 'Chrome/49.0.2587.3',
            'Accept' => 'application/json',
            'Accept-Encoding' => 'gzip',
        ]);

        $upgraded = $client->upgrade('/');

        if (!$upgraded) {
            \Log::info('WS Client process: failed to connect');
            return;
        }

        \Log::info('WS Client process: connected');
        while (!self::$quit) {
            \Log::debug(
                'WS Client process: is listening',
                self::getProcessDetails($client, $process, $swoole)
            );
            $frame = $client->recv(20);
            if ($client->errCode) {
                \Log::error($client->errMsg, self::getProcessDetails($client, $process, $swoole));
                $client->close();
                $process->exit($client->errCode);
                self::$quit = true;
            }
            if ($frame instanceof Frame && $frame->data !== null) {
                \Log::debug('WS Client process: received frame', (array) $frame);
                self::processFrame($frame, $process);
                Coroutine::sleep(0.5);
            }
        }
    }

    public static function getProcessDetails(Client $client, Process $process, Server $server): array
    {
        return [
            'body' => $client->getBody(),
            'connected' => $client->connected,
            'errorCode' => $client->errCode,
            'errorMsg' => $client->errMsg,
            'statusCode' => $client->statusCode,
            'socketName' => $client->getsockname(),
            'peerName' => $client->getpeername(),
            'pid' => $process->pid,
            'id' => $process->id,
            'queueId' => $process->msgQueueId,
            'queueKey' => $process->msgQueueKey,
            'lastServerError' => $server->getLastError(),
        ];
    }

    public static function processFrame(Frame $frame, Process $process): void
    {
        try {
           $data = json_decode($frame->data, true, 512, JSON_THROW_ON_ERROR | JSON_OBJECT_AS_ARRAY);
           \Log::debug('Message: ', $data);
        } catch (\JsonException $jsonException) {
            \Log::error(
                $frame->data,
                [
                    'previousError' => $jsonException->getMessage()
                        . ' at ' . $jsonException->getFile() .
                        ':' . $jsonException->getLine(),
                ]
            );
            $process->exit($jsonException->getCode());
        }
    }

    public static function onReload(Server $swoole, Process $process): void
    {
        \Log::info('WS Client process: reloading');
        self::$quit = true;
    }

    public static function onStop(Server $swoole, Process $process): void
    {
        \Log::info(
            'WS Client process: stopped',
            [
                $swoole->getLastError(),
            ]
        );
        self::$quit = true;
        $process->exit(0);
    }
}

What is the best approach?