search

phalcon / cphalcon

High performance, full-stack PHP framework delivered as a C extension.
https://phalcon.io
BSD 3-Clause "New" or "Revised" License
10.78k stars 1.96k forks source link

Segmentation fault happens randomly at high traffic ( zend_objects_store_put ) #2948

Closed bancuadrian closed 9 years ago

bancuadrian commented 9 years ago

Ubuntu 14.04 LTS PHP 5.5.9 Apache 2.4.7

Apache coredump with gdb :

Core was generated by `/usr/sbin/apache2 -k start'. Program terminated with signal SIGSEGV, Segmentation fault.

0 0x00007f5fea6128c6 in zend_objects_store_put (object=0x7f5ff052e160, dtor=0x7f5fea60cb00 <zend_objects_destroy_object>, free_storage=0x7f5fe28830f0 <phalcon_config_object_dtor>, clone=0x0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c:114

114 /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c: No such file or directory.

BT FULL

Core was generated by `/usr/sbin/apache2 -k start'. Program terminated with signal SIGSEGV, Segmentation fault.

0 0x00007f5fea6128c6 in zend_objects_store_put (object=0x7f5ff052e160, dtor=0x7f5fea60cb00 <zend_objects_destroy_object>, free_storage=0x7f5fe28830f0 <phalcon_config_object_dtor>, clone=0x0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c:114

114 /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c: No such file or directory. (gdb) bt full

0 0x00007f5fea6128c6 in zend_objects_store_put (object=0x7f5ff052e160, dtor=0x7f5fea60cb00 <zend_objects_destroy_object>, free_storage=0x7f5fe28830f0 <phalcon_config_object_dtor>, clone=0x0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c:114

    handle = 124
    obj = &lt;optimized out>

1 0x00007f5fe2882f6d in phalcon_config_object_ctor () from /usr/lib/php5/20121212/phalcon.so

No symbol table info available.

2 0x00007f5fea5ee4a7 in _object_and_properties_init (arg=arg@entry=0x7f5ff052e000, class_type=0x7f5ff0c3c200, properties=properties@entry=0x0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_API.c:1200

    object = 0x7f5f94a70eb0

3 0x00007f5fea5ee537 in _object_init_ex (arg=arg@entry=0x7f5ff052e000, class_type=<optimized out>) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_API.c:1208

No locals.

4 0x00007f5fea640f46 in ZEND_NEW_SPEC_HANDLER (execute_data=0x7f5ff050d210) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_vm_execute.h:758

    opline = 0x7f5f94a70eb0
    object_zval = 0x7f5ff052e000
    constructor = &lt;optimized out>

5 0x00007f5fea6149e8 in execute_ex (execute_data=0x7f5ff050d210) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_vm_execute.h:363

    ret = &lt;optimized out>
    original_in_execution = 1 '\001'

6 0x00007f5fea5dab59 in dtrace_execute_ex (execute_data=<optimized out>) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_dtrace.c:73

    lineno = &lt;optimized out>
    scope = 0x0
    filename = &lt;optimized out>
    funcname = &lt;optimized out>
    classname = &lt;optimized out>

7 0x00007f5fea69a7bf in ZEND_INCLUDE_OR_EVAL_SPEC_CONST_HANDLER (execute_data=0x7f5ff050d0d0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_vm_execute.h:2748

    opline = 0x7f5f94a6e628
    new_op_array = 0x7f5ff052d398
    inc_filename = 0x7f5f94a6e1d8
    tmp_inc_filename = 0x0
    failure_retval = 0 '\000'

8 0x00007f5fea6149e8 in execute_ex (execute_data=0x7f5ff050d0d0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_vm_execute.h:363

    ret = &lt;optimized out>
    original_in_execution = 0 '\000'

9 0x00007f5fea5dab59 in dtrace_execute_ex (execute_data=<optimized out>) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_dtrace.c:73

    lineno = &lt;optimized out>
    scope = 0x0
    filename = &lt;optimized out>
    funcname = &lt;optimized out>
    classname = &lt;optimized out>

10 0x00007f5fea5ec5e0 in zend_execute_scripts (type=type@entry=2, retval=retval@entry=0x0, file_count=file_count@entry=1) at /build/buildd/php5-5.5.9+dfsg/Zend/zend.c:1316

    files = {{gp_offset = 32, fp_offset = 32607, overflow_arg_area = 0x7fff4261b570, reg_save_area = 0x7fff4261b500}}
    i = 0
    file_handle = 0x7fff4261b5a0
    orig_op_array = 0x0
    orig_retval_ptr_ptr = 0x0
    orig_interactive = 0

11 0x00007f5fea69c4ed in php_handler (r=<optimized out>) at /build/buildd/php5-5.5.9+dfsg/sapi/apache2handler/sapi_apache2.c:669

    zfd = {type = ZEND_HANDLE_FILENAME, filename = 0x7f5fee9a1800 "/var/www/vhosts/delivery/current/public/index.php", opened_path = 0x0, handle = {fd = 1113700060, fp = 0x7fff4261b6dc, stream = {handle = 0x7fff4261b6dc, isatty = -291891200, mmap = {
            len = 140049901037808, pos = 140050001698360, map = 0x7f5f00000000, buf = 0x7f5feea85bc8 "", old_handle = 0x7f5feea91748, old_closer = 0x7f5feea91748}, reader = 0x7f5fee9a1d00, fsizer = 0x7f5fee9a5f00, closer = 0x7f5fee9df290}}, free_filename = 0 '\000'}
    __orig_bailout = 0x0
    __bailout = {{__jmpbuf = {140050001714944, -7792186829664746521, 140050001714944, 140050006232960, 0, 140050001949328, 7791811881353544679, 7882218056389297127}, __mask_was_saved = 0, __saved_mask = {__val = {140050003104728, 140050002752280, 140050001692232, 1,
            140050002752280, 140050001692240, 140050001765624, 140050006232960, 0, 140050001949328, 140049949641891, 140050001696817, 2, 140050001692240, 140050001714944, 140050002624072}}}}
    ctx = 0x7f5fee9be690
    conf = &lt;optimized out>
    brigade = 0x7f5fee9c2c50
    bucket = &lt;optimized out>
    rv = &lt;optimized out>
    parent_req = 0x7f5fee9d3fb8

12 0x00007f5feebb0680 in ap_run_handler (r=0x7f5fee9a5f00) at config.c:169

    pHook = 0x7f5feea83e70

---Type <return> to continue, or q <return> to quit--- n = 2 rv = 124

13 0x00007f5feebb0bc9 in ap_invoke_handler (r=r@entry=0x7f5fee9a5f00) at config.c:439

    handler = &lt;optimized out>
    p = &lt;optimized out>
    result = &lt;optimized out>
    old_handler = 0x7f5feea9fbe8 "application/x-httpd-php"
    ignore = &lt;optimized out>

14 0x00007f5feebc5c2c in ap_internal_redirect (new_uri=<optimized out>, r=<optimized out>) at http_request.c:644

    new = 0x7f5fee9a5f00
    access_status = &lt;optimized out>

15 0x00007f5fe899acfc in handler_redirect (r=0x7f5fee9b24f8) at mod_rewrite.c:5063

No locals.

16 0x00007f5feebb0680 in ap_run_handler (r=0x7f5fee9b24f8) at config.c:169

    pHook = 0x7f5feea83e98
    n = 3
    rv = 124

17 0x00007f5feebb0bc9 in ap_invoke_handler (r=r@entry=0x7f5fee9b24f8) at config.c:439

    handler = &lt;optimized out>
    p = &lt;optimized out>
    result = &lt;optimized out>
    old_handler = 0x7f5fe89a31e9 "redirect-handler"
    ignore = &lt;optimized out>

18 0x00007f5feebc5c2c in ap_internal_redirect (new_uri=<optimized out>, r=<optimized out>) at http_request.c:644

    new = 0x7f5fee9b24f8
    access_status = &lt;optimized out>

19 0x00007f5fe899acfc in handler_redirect (r=0x7f5fee9b50a0) at mod_rewrite.c:5063

No locals.

20 0x00007f5feebb0680 in ap_run_handler (r=0x7f5fee9b50a0) at config.c:169

    pHook = 0x7f5feea83e98
    n = 3
    rv = 124

21 0x00007f5feebb0bc9 in ap_invoke_handler (r=r@entry=0x7f5fee9b50a0) at config.c:439

    handler = &lt;optimized out>
    p = &lt;optimized out>
    result = &lt;optimized out>
    old_handler = 0x7f5fe89a31e9 "redirect-handler"
    ignore = &lt;optimized out>

22 0x00007f5feebc616a in ap_process_async_request (r=0x7f5fee9b50a0) at http_request.c:317

    access_status = 0

23 0x00007f5feebc6444 in ap_process_request (r=r@entry=0x7f5fee9b50a0) at http_request.c:363

    bb = &lt;optimized out>
    b = &lt;optimized out>
    c = 0x7f5fee9df290
    rv = &lt;optimized out>

24 0x00007f5feebc2f02 in ap_process_http_sync_connection (c=0x7f5fee9df290) at http_core.c:190

    r = 0x7f5fee9b50a0
    cs = 0x0
    csd = 0x7f5fee9df0a0
    mpm_state = 1

25 ap_process_http_connection (c=0x7f5fee9df290) at http_core.c:231

No locals.

26 0x00007f5feebb9cc0 in ap_run_process_connection (c=0x7f5fee9df290) at connection.c:41

    pHook = 0x7f5feea7c440
    n = 0
    rv = 124

27 0x00007f5feebba0a8 in ap_process_connection (c=c@entry=0x7f5fee9df290, csd=<optimized out>) at connection.c:202

    rc = &lt;optimized out>

---Type <return> to continue, or q <return> to quit---

28 0x00007f5feafbc767 in child_main (child_num_arg=child_num_arg@entry=197) at prefork.c:704

    current_conn = 0x7f5fee9df290
    csd = 0x7f5fee9df0a0
    thd = 0x7f5fee9e10a0
    osthd = 140050003421056
    ptrans = 0x7f5fee9df028
    allocator = 0x7f5ff0e9c390
    status = &lt;optimized out>
    i = &lt;optimized out>
    lr = &lt;optimized out>
    pollset = 0x7f5fee9e1158
    sbh = 0x7f5fee9e1150
    bucket_alloc = 0x7f5fee9db028
    last_poll_idx = 1
    lockfile = &lt;optimized out>

29 0x00007f5feafbc9a6 in make_child (s=0x7f5feeb1ede0, slot=197) at prefork.c:800

    pid = 0

30 0x00007f5feafbd60e in perform_idle_server_maintenance (p=<optimized out>) at prefork.c:902

    i = &lt;optimized out>
    idle_count = &lt;optimized out>
    ws = &lt;optimized out>
    free_length = &lt;optimized out>
    free_slots = {189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220}
    last_non_dead = &lt;optimized out>
    total_non_dead = &lt;optimized out>

31 prefork_run (_pconf=<optimized out>, plog=<optimized out>, s=<optimized out>) at prefork.c:1090

    status = 32607
    pid = {pid = -1, in = 0x7f5feebd1048, out = 0xa, err = 0x7f5fee2afff6}
    child_slot = &lt;optimized out>
    exitwhy = (APR_PROC_SIGNAL | APR_PROC_SIGNAL_CORE | unknown: 3995795440)
    processed_status = &lt;optimized out>
    index = &lt;optimized out>
    remaining_children_to_start = 0
    rv = &lt;optimized out>

32 0x00007f5feeb9769e in ap_run_mpm (pconf=0x7f5feeb4d028, plog=0x7f5feeb1a028, s=0x7f5feeb1ede0) at mpm_common.c:96

    pHook = 0x7f5feea7c978
    n = 0
    rv = 124

33 0x00007f5feeb90e36 in main (argc=3, argv=0x7fff4261bde8) at main.c:777

    c = 0 '\000'
    showcompile = 0
    showdirectives = 0
    confname = 0x7f5feebd0607 "apache2.conf"
    def_server_root = 0x7f5feebd05fa "/etc/apache2"
    temp_error_log = 0x0
    error = &lt;optimized out>
    process = 0x7f5feeb4f118
    pconf = 0x7f5feeb4d028
    plog = 0x7f5feeb1a028
    ptemp = 0x7f5feeb1c028
    pcommands = 0x7f5feeb24028
    opt = 0x7f5feeb24118
    rv = &lt;optimized out>
    mod = 0x7f5feedf2160 &lt;ap_prelinked_modules+64>
    opt_arg = 0x7f5feeb4f028 "(0\265\356_\177"
    signal_server = &lt;optimized out>

[PHP Modules] bcmath bz2 calendar Core ctype date dba dom ereg exif FiftyOne_Degrees_Detector fileinfo filter ftp geoip gettext hash iconv intl json libxml mbstring mcrypt memcache memcached mhash mysql mysqli openssl pcntl pcre PDO pdo_mysql phalcon Phar posix readline recode Reflection session shmop SimpleXML soap sockets SPL standard sysvmsg sysvsem sysvshm tokenizer wddx xml xmlreader xmlrpc xmlwriter xsl Zend OPcache zip zlib

[Zend Modules] Zend OPcache

Apache Compiled in modules: core.c mod_so.c mod_watchdog.c http_core.c mod_log_config.c mod_logio.c mod_version.c mod_unixd.c

Want to back this issue? Post a bounty on it! We accept bounties via Bountysource.

bancuadrian commented 9 years ago

Can someone help in some way?

oh-ren commented 9 years ago

I am certainly no expert in this regard, but I can try to do some suggestions of course. Anyhow, you didn't mention the Phalcon ver. you're running? Are you running with the latest version, i.e. 1.3.4? Upgrading PHP to the latest 5.5.x might help? There are a lot of reported vague issues related to segmentation faults. It seems for certain issues it helps to disable OPcache. Not ideal I reckon, but it might help. (I'd first try to ensure you're running the latest version(s). I also remember someone btw who moved back to 1.2.6 because of 'weird' issues with 1.3.x..) (of course that's not a solution, because there's not really a 1.2.x branch, 1.3.0 is the successor of 1.2.6)

bancuadrian commented 9 years ago

Phalcon version is 1.3.3, but we tried all versions between 1.2.6 and 1.3.4. Also we already tried disabling OPcache on all Phalcon versions with no luck. Issue still happen.

oh-ren commented 9 years ago

Is it possible for you to upgrade PHP?

bancuadrian commented 9 years ago

We already tried with 5.5.9 and 5.5.17 and of course all Phalcon versions

oh-ren commented 9 years ago

Seeing reports on stuff breaking at high volume, and other reports on seemingly random segfaults, do scare me a bit (much).

dompie commented 9 years ago

Let me go on with suggestions: The backtrace says at first (exit) point: 

#0 0x00007f5fea6128c6 in zend_objects_store_put (object=0x7f5ff052e160, dtor=0x7f5fea60cb00 <zend_objects_destroy_object>, free_storage=0x7f5fe28830f0 <phalcon_config_object_dtor>, clone=0x0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c:114
handle = 124

And line 114 said:

"114 /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c: No such file or directory."

Maybe there have been some .c files removed after compilation which are still required? Maybe some (op) cache files have been removed from outside php and php still thinks, they are present (or vice versa)? Maybe a directory could not be created, because disk runs full in some rare cases?

dugwood commented 9 years ago

@dompie: this isn't the issue, the gdb tries to load the file where the issue happens. Also you never run .c/.h/.zep files, which are not compiled, so not runnable.

I'm experiencing some segmentation fault too, the last one in phalcon 2: https://github.com/phalcon/cphalcon/pull/3026#issuecomment-62923039

For now, I've managed to go around each bug, but that's difficult to go forward with some of these tricky bugs.

@renskii: I know some high traffic websites that run phalcon 1.3.4 without any issue. Maybe the issue is the webserver, in your cases, or in mine. PHP wasn't segfaulting when I was running the CLI version, when the same script with the CGI version was segfaulting (I use lighttpd+fastcgi).

@bancuadrian: can you try the same thing under nginx/php-fpm or something like that? Maybe that would help. Another hint: https://bugs.php.net/bug.php?id=67089

denniscuijpers commented 9 years ago

@bancuadrian Don't know if you still have issues with php segfaulting. I had exactly the same backtraces and finally found a solution by first disabling php opcache to get a better stacktrace, turns out it has nothing to do with the script itself.

The problem is caused by apache not correctly handing multiple requests / http pipelining through a single connection. Turning KeepAlive off did the trick for me. See https://bugs.php.net/bug.php?id=68486 and https://issues.apache.org/bugzilla/show_bug.cgi?id=56984

Hope this helps!

scresp0 commented 9 years ago

Hi,

I'm running Ubuntu with nginx + wordpress +php5-fpm and I'm having this issue just with some posts or web pages on my installation..

..and the most amazing thing is, that it doesn't happen fetching these URLs from a mobile device (it uses a mobile theme then).. Debugging what's going on:

For instance, I access this URL from my computer web browser and the php5 execution crashes, so the reader gets a 502 error.

http://sophosiberia.es/5-sencillos-consejos-para-evitar-ransomware/ or http://sophosiberia.es/boletin-seguridad/

If you open any of these two URLs from a mobile phone, they just work!

So taking a look at /var/log/apport.log I can just read: ERROR: apport (pid 8659) Fri Mar 27 18:00:11 2015: called for pid 7445, signal 11, core limit 0 ERROR: apport (pid 8659) Fri Mar 27 18:00:11 2015: executable: /usr/sbin/php5-fpm (command line "php-fpm:\ pool\ www") ERROR: apport (pid 8659) Fri Mar 27 18:00:11 2015: is_closing_session(): no DBUS_SESSION_BUS_ADDRESS in environment ERROR: apport (pid 8659) Fri Mar 27 18:00:37 2015: wrote report /var/crash/_usr_sbin_php5-fpm.33.crash

So, let's go: Let's have a look at that crash file:

cd /var/crash apport-cli ./_usr_sbin_php5-fpm.33.crash

*\ Send problem report to the developers?

After the problem report has been sent, please fill out the form in the automatically opened web browser.

What would you like to do? Your options are: S: Send report (10.4 MB) E: Examine locally V: View report K: Keep report file for sending later or copying to somewhere else I: Cancel and ignore future crashes of this program version C: Cancel Please choose (S/E/V/K/I/C): E *\ Collecting problem information

The collected information can be sent to the developers to improve the application. This might take a few minutes. ......................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................

and so.. takes a while, and at the end:

For help, type "help". Type "apropos word" to search for commands related to "word". Reading symbols from /tmp/apport_sandbox_IvosRM//usr/sbin/php5-fpm...Reading symbols from /tmp/apport_sandbox_IvosRM/usr/lib/debug/usr/sbin/php5-fpm...done. done. [New LWP 7450] warning: Could not load shared library symbols for /usr/lib/php5/20121212/newrelic.so. Do you need "set solib-search-path" or "set sysroot"? [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". warning: the debug information found in "/tmp/apport_sandbox_IvosRM/usr/lib/debug/usr/lib/php5/20121212/mysql.so" does not match "/tmp/apport_sandbox_IvosRM/usr/lib/php5/20121212/mysql.so" (CRC mismatch).

warning: the debug information found in "/tmp/apport_sandbox_IvosRM/usr/lib/debug/usr/lib/php5/20121212/mysqli.so" does not match "/tmp/apport_sandbox_IvosRM/usr/lib/php5/20121212/mysqli.so" (CRC mismatch).

warning: the debug information found in "/tmp/apport_sandbox_IvosRM/usr/lib/debug/usr/lib/php5/20121212/pdo_mysql.so" does not match "/tmp/apport_sandbox_IvosRM/usr/lib/php5/20121212/pdo_mysql.so" (CRC mismatch).

Core was generated by `php-fpm: pool www '. Program terminated with signal SIGSEGV, Segmentation fault.

0 0x00000000006efb64 in zend_get_class_entry (zobject=0x334fb48) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_API.c:237

237 /build/buildd/php5-5.5.9+dfsg/Zend/zend_API.c: No such file or directory.

Same thing commented here.. But I have no idea what it means, and why it happens. Any ideas?

thanks in advance!

andresgutierrez commented 9 years ago

Could you please try again using Phalcon 2 or close this issue as 1.3.x is no longer supported