Crashing Randomly On Ubuntu 14.04 Server With nginx/1.7.5

[war59312]

Hi,

New to this whole hhvm thing.. Trying to figure it out.

Keeps crashing on me randomly. :(

Ubuntu 14.04 Server with nginx/1.7.5

Running normal build from https://github.com/facebook/hhvm/wiki/Prebuilt-packages-on-Ubuntu-14.04

Guess I will have to try that debug build when I have time.

Easy way to crash is running ab:

rob@RobsUbuntuServer:~/Desktop$ ab -c 50 -n 1000 http://localhost/
This is ApacheBench, Version 2.3 <$Revision: 1528965 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking localhost (be patient)
Completed 100 requests
Completed 200 requests
Completed 300 requests
Completed 400 requests
Completed 500 requests
Completed 600 requests
Completed 700 requests
Completed 800 requests
Completed 900 requests
Completed 1000 requests
Finished 1000 requests

Server Software:        nginx/1.7.5
Server Hostname:        localhost
Server Port:            80

Document Path:          /
Document Length:        7838 bytes

Concurrency Level:      50
Time taken for tests:   3.485 seconds
Complete requests:      1000
Failed requests:        747
   (Connect: 0, Receive: 0, Length: 747, Exceptions: 0)
Non-2xx responses:      737
Total transferred:      2368361 bytes
HTML transferred:       2188168 bytes
Requests per second:    286.93 [#/sec] (mean)
Time per request:       174.257 [ms] (mean)
Time per request:       3.485 [ms] (mean, across all concurrent requests)
Transfer rate:          663.63 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    1   0.7      0       3
Processing:     1  173 259.6      5     805
Waiting:        1  173 259.4      5     805
Total:          3  174 259.3      5     805
WARNING: The median and mean for the initial connection time are not within a normal deviation
        These results are probably not that reliable.

Percentage of the requests served within a certain time (ms)
  50%      5
  66%      8
  75%    492
  80%    529
  90%    601
  95%    655
  98%    692
  99%    733
 100%    805 (longest request)

Crashes every single time after this.

php.ini:

; php options

; hhvm specific 
hhvm.log.level = Warning
hhvm.log.always_log_unhandled_exceptions = true
hhvm.log.runtime_error_reporting_level = 8191
hhvm.mysql.typed_results = false
hhvm.eval.jit_warmup_requests = 0
hhvm.eval.jit = true

sever.ini

pid = /var/run/hhvm/pid
 hhvm.server.file_socket=/var/run/hhvm/hhvm.sock
 hhvm.server.type = fastcgi
 hhvm.server.default_document = index.php
 hhvm.log.level = Warning
 hhvm.log.always_log_unhandled_exceptions = true
 hhvm.log.runtime_error_reporting_level = 8191
 hhvm.log.use_log_file = true
 hhvm.log.file = /var/log/hhvm/error.log
 hhvm.repo.central.path = /var/run/hhvm/hhvm.hhbc
 hhvm.mysql.typed_results = false
 hhvm.eval.jit_warmup_requests = 0

Does not seem to matter if run via sock or normal. Crashes every time with either settings.

; php options

pid = /var/run/hhvm/pid

; hhvm specific 

hhvm.server.port = 9000
hhvm.server.type = fastcgi
hhvm.server.default_document = index.php
hhvm.log.use_log_file = true
hhvm.log.file = /var/log/hhvm/error.log
hhvm.repo.central.path = /var/run/hhvm/hhvm.hhbc
hhvm.eval.jit_warmup_requests = 0
hhvm.eval.jit = true

nginx config:

# You may add here your
# server {
#   ...
# }
# statements for each of your virtual hosts to this file

##
# You should look at the following URL's in order to grasp a solid understanding
# of Nginx configuration files in order to fully unleash the power of Nginx.
# http://wiki.nginx.org/Pitfalls
# http://wiki.nginx.org/QuickStart
# http://wiki.nginx.org/Configuration
#
# Generally, you will want to move this file somewhere, and start with a clean
# file but keep this around for reference. Or just disable in sites-enabled.
#
# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples.
##

server {
    listen 80 default_server;
    listen [::]:80 default_server ipv6only=on;

    root /usr/share/nginx/html;
    index index.php index.html index.htm;

    # Make site accessible from http://localhost/
    server_name localhost;

    # access_log  /var/log/nginx/localhost.access.log;
    access_log off;
    error_log /var/log/nginx/localhost.error.log;

    # Enable HHVM Facebook PHP Stack
        # include hhvm.conf;

    location ~ \.php$ {
            fastcgi_pass unix:/var/run/hhvm/hhvm.sock;

            fastcgi_index   index.php;
            fastcgi_param   SCRIPT_FILENAME  $document_root$fastcgi_script_name;
            include         fastcgi_params;
         }

    # redis - not working so it seems
    # location /index.php {
        # alias /usr/share/nginx/html/wp-index-redis.php;
    # }

    location / {
        # First attempt to serve request as file, then
        # as directory, then fall back to displaying a 404.
        # try_files $uri $uri/ =404;

        # For Wordpress Clean URLs
        try_files $uri $uri/ /index.php?$args;

    # redis - not working get a blank white page        
    # index wp-index-redis.php;
        #  try_files $uri $uri/ /wp-index-redis.php?$args;
        # if (-e $request_filename) {
        # break;
        # }
        # rewrite ^/(.+)$ /index.php?q=$1 last;
    }   

        # Munin FastCGI Graphs
    location ^~ /munin-cgi/munin-cgi-graph/ {
        fastcgi_split_path_info ^(/munin-cgi/munin-cgi-graph)(.*);
        fastcgi_param PATH_INFO $fastcgi_path_info;
        fastcgi_pass unix:/var/run/munin/fastcgi-graph.sock;
        include fastcgi_params;
        }

        # Munin FastCGI HTML
    location  ^~ /munin-cgi/munin-cgi-html/ {
        fastcgi_split_path_info ^(/munin-cgi/munin-cgi-html)(.*);
        fastcgi_param PATH_INFO $fastcgi_path_info;
        fastcgi_pass unix:/var/run/munin/fastcgi-html.sock;
        include fastcgi_params;
        }   

        # Munin Static HTML Files
    location /munin/static/ {
        alias /var/www/munin/static/;
        expires modified +1w;
    }

        # Munin HTML Files
        location /munin/ {
        alias /var/www/munin/;
        expires modified +310s;
    }

    # Disable viewing .htaccess & .htpassword
    location ~ /\.ht {
            deny  all;
    }

    #error_page 404 /404.html;

    # redirect server error pages to the static page /50x.html
    #
    #error_page 500 502 503 504 /50x.html;
    #location = /50x.html {
    #   root /usr/share/nginx/html;
    #}

    # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
    #
    #location ~ \.php$ {
    #   fastcgi_split_path_info ^(.+\.php)(/.+)$;
    #   # NOTE: You should have "cgi.fix_pathinfo = 0;" in php.ini
    #
    #   # With php5-cgi alone:
    #   fastcgi_pass 127.0.0.1:9000;
    #   # With php5-fpm:
    #   fastcgi_pass unix:/var/run/php5-fpm.sock;
    #   fastcgi_index index.php;
    #   include fastcgi_params;
    #}

    # deny access to .htaccess files, if Apache's document root
    # concurs with nginx's one
    #
    #location ~ /\.ht {
    #   deny all;
    #}

# BEGIN W3TC Minify cache
location ~ /wp-content/cache/minify.*\.js$ {
    types {}
    default_type application/x-javascript;
}
location ~ /wp-content/cache/minify.*\.css$ {
    types {}
    default_type text/css;
}
# END W3TC Minify cache
# BEGIN W3TC Page Cache cache
location ~ /wp-content/cache/page_enhanced.*html$ {
    add_header Vary Cookie;
}
# END W3TC Page Cache cache
# BEGIN W3TC Minify core
rewrite ^/wp-content/cache/minify.*/w3tc_rewrite_test$ /wp-content/plugins/w3-total-cache/pub/minify.php?w3tc_rewrite_test=1 last;
set $w3tc_enc "";
if (-f $request_filename$w3tc_enc) {
    rewrite (.*) $1$w3tc_enc break;
}
rewrite ^/wp-content/cache/minify/(.+/[X]+\.css)$ /wp-content/plugins/w3-total-cache/pub/minify.php?test_file=$1 last;
rewrite ^/wp-content/cache/minify/(.+\.(css|js))$ /wp-content/plugins/w3-total-cache/pub/minify.php?file=$1 last;
# END W3TC Minify core
# BEGIN W3TC Page Cache core
rewrite ^(.*\/)?w3tc_rewrite_test/?$ $1?w3tc_rewrite_test=1 last;
set $w3tc_rewrite 1;
if ($request_method = POST) {
    set $w3tc_rewrite 0;
}
if ($query_string != "") {
    set $w3tc_rewrite 0;
}
if ($request_uri !~ \/$) {
    set $w3tc_rewrite 0;
}
if ($http_cookie ~* "(comment_author|wp\-postpass|w3tc_logged_out|wordpress_logged_in)") {
    set $w3tc_rewrite 0;
}
set $w3tc_ua "";
if ($http_user_agent ~* "(2\.0\ mmp|240x320|alcatel|amoi|asus|au\-mic|audiovox|avantgo|benq|bird|blackberry|blazer|cdm|cellphone|danger|ddipocket|docomo|dopod|elaine/3\.0|ericsson|eudoraweb|fly|haier|hiptop|hp\.ipaq|htc|huawei|i\-mobile|iemobile|j\-phone|kddi|konka|kwc|kyocera/wx310k|lenovo|lg|lg/u990|lge\ vx|midp|midp\-2\.0|mmef20|mmp|mobilephone|mot\-v|motorola|netfront|newgen|newt|nintendo\ ds|nintendo\ wii|nitro|nokia|novarra|o2|openweb|opera\ mobi|opera\.mobi|palm|panasonic|pantech|pdxgw|pg|philips|phone|playstation\ portable|portalmmm|ppc|proxinet|psp|pt|qtek|sagem|samsung|sanyo|sch|sec|sendo|sgh|sharp|sharp\-tq\-gx10|small|smartphone|softbank|sonyericsson|sph|symbian|symbian\ os|symbianos|toshiba|treo|ts21i\-10|up\.browser|up\.link|uts|vertu|vodafone|wap|willcome|windows\ ce|windows\.ce|winwap|xda|zte)") {
    set $w3tc_ua _low;
}
if ($http_user_agent ~* "(acer\ s100|android|archos5|blackberry9500|blackberry9530|blackberry9550|cupcake|docomo\ ht\-03a|dream|htc\ hero|htc\ magic|htc_dream|htc_magic|incognito|ipad|iphone|ipod|lg\-gw620|liquid\ build|maemo|mot\-mb200|mot\-mb300|nexus\ one|opera\ mini|samsung\-s8000|series60.*webkit|series60/5\.0|sonyericssone10|sonyericssonu20|sonyericssonx10|t\-mobile\ mytouch\ 3g|t\-mobile\ opal|tattoo|webmate|webos)") {
    set $w3tc_ua _high;
}
if ($http_cookie ~* "(w3tc_preview)") {
    set $w3tc_rewrite _preview;
}
set $w3tc_ref "";
if ($http_cookie ~* "w3tc_referrer=.*(google\.com|yahoo\.com|bing\.com|ask\.com|msn\.com)") {
    set $w3tc_ref _search_engines;
}
set $w3tc_ssl "";
if ($scheme = https) {
    set $w3tc_ssl _ssl;
}
set $w3tc_ext "";
if (-f "$document_root/wp-content/cache/page_enhanced/$http_host/$request_uri/_index$w3tc_ua$w3tc_ref$w3tc_ssl$w3tc_rewrite.html") {
    set $w3tc_ext .html;
}
if (-f "$document_root/wp-content/cache/page_enhanced/$http_host/$request_uri/_index$w3tc_ua$w3tc_ref$w3tc_ssl$w3tc_rewrite.xml") {
    set $w3tc_ext .xml;
}
if ($w3tc_ext = "") {
  set $w3tc_rewrite 0;
}
if ($w3tc_rewrite = 1) {
    rewrite .* "/wp-content/cache/page_enhanced/$http_host/$request_uri/_index$w3tc_ua$w3tc_ref$w3tc_ssl$w3tc_rewrite$w3tc_ext" last;
}
# END W3TC Page Cache core

}

# another virtual host using mix of IP-, name-, and port-based configuration
#
#server {
#   listen 8000;
#   listen somename:8080;
#   server_name somename alias another.alias;
#   root html;
#   index index.html index.htm;
#
#   location / {
#       try_files $uri $uri/ =404;
#   }
#}

# HTTPS server
#
#server {
#   listen 443;
#   server_name localhost;
#
#   root html;
#   index index.html index.htm;
#
#   ssl on;
#   ssl_certificate cert.pem;
#   ssl_certificate_key cert.key;
#
#   ssl_session_timeout 5m;
#
#   ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
#   ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
#   ssl_prefer_server_ciphers on;
#
#   location / {
#       try_files $uri $uri/ =404;
#   }
#}

Thanks,

Will

[ptarjan]

Can you try running the hhvm-dbg package instead? And look for /tmp/stacktrace.*.log and paste those.

[war59312]

Will do. But I norrowed it down to an isssue with W3 Total Cache.

It's what is casuing the crash. See: https://wordpress.org/support/topic/warning-with-hhvm

[williejackson]

@war59312 Thanks for bringing this to our attention, we took a look and verified the issue being reported. A fix will be in the forthcoming release.

[war59312]

Nice, glad to hear it. Thanks.

Was going to get that long for you this weekend but sounds like you dont relaly need it.

[tillkruss]

Same issue here. Nginx & PHP5-FPM works. When I use HHVM 3.2.0 I get the following error:

Warning: Parameter 1 to W3_Plugin_TotalCache::ob_callback() expected to be a reference, value given in /wp-includes/functions.php on line 3092

[pjv]

anyone have a short-term patch to fix this?

[paulbiss]

I can't reproduce the crash locally, does anyone have a stacktrace?

[paulbiss]

Closing due to inactivity.

https://github.com/facebook/hhvm/wiki/Human-Timeouts

[kevin25]

i got the same issue. And HHVM was crashed.