Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Segmentation fault happens randomly at high traffic ( zend_objects_store_put ) #2948

Closed
baad777 opened this issue Oct 28, 2014 · 11 comments
Closed

Segmentation fault happens randomly at high traffic ( zend_objects_store_put ) #2948

baad777 opened this issue Oct 28, 2014 · 11 comments

Comments

@baad777
Copy link

baad777 commented Oct 28, 2014

Ubuntu 14.04 LTS
PHP 5.5.9
Apache 2.4.7

Apache coredump with gdb :

Core was generated by `/usr/sbin/apache2 -k start'. Program terminated with signal SIGSEGV, Segmentation fault. #0 0x00007f5fea6128c6 in zend_objects_store_put (object=0x7f5ff052e160, dtor=0x7f5fea60cb00 <zend_objects_destroy_object>, free_storage=0x7f5fe28830f0 <phalcon_config_object_dtor>, clone=0x0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c:114

114 /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c: No such file or directory.

BT FULL

Core was generated by `/usr/sbin/apache2 -k start'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x00007f5fea6128c6 in zend_objects_store_put (object=0x7f5ff052e160, dtor=0x7f5fea60cb00 <zend_objects_destroy_object>, free_storage=0x7f5fe28830f0 <phalcon_config_object_dtor>, clone=0x0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c:114

114 /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c: No such file or directory.
(gdb) bt full
#0 0x00007f5fea6128c6 in zend_objects_store_put (object=0x7f5ff052e160, dtor=0x7f5fea60cb00 <zend_objects_destroy_object>, free_storage=0x7f5fe28830f0 <phalcon_config_object_dtor>, clone=0x0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c:114

    handle = 124
    obj = &lt;optimized out>

#1 0x00007f5fe2882f6d in phalcon_config_object_ctor () from /usr/lib/php5/20121212/phalcon.so

No symbol table info available.
#2 0x00007f5fea5ee4a7 in _object_and_properties_init (arg=arg@entry=0x7f5ff052e000, class_type=0x7f5ff0c3c200, properties=properties@entry=0x0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_API.c:1200

    object = 0x7f5f94a70eb0

#3 0x00007f5fea5ee537 in _object_init_ex (arg=arg@entry=0x7f5ff052e000, class_type=<optimized out>) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_API.c:1208

No locals.
#4 0x00007f5fea640f46 in ZEND_NEW_SPEC_HANDLER (execute_data=0x7f5ff050d210) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_vm_execute.h:758

    opline = 0x7f5f94a70eb0
    object_zval = 0x7f5ff052e000
    constructor = &lt;optimized out>

#5 0x00007f5fea6149e8 in execute_ex (execute_data=0x7f5ff050d210) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_vm_execute.h:363

    ret = &lt;optimized out>
    original_in_execution = 1 '\001'

#6 0x00007f5fea5dab59 in dtrace_execute_ex (execute_data=<optimized out>) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_dtrace.c:73

    lineno = &lt;optimized out>
    scope = 0x0
    filename = &lt;optimized out>
    funcname = &lt;optimized out>
    classname = &lt;optimized out>

#7 0x00007f5fea69a7bf in ZEND_INCLUDE_OR_EVAL_SPEC_CONST_HANDLER (execute_data=0x7f5ff050d0d0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_vm_execute.h:2748

    opline = 0x7f5f94a6e628
    new_op_array = 0x7f5ff052d398
    inc_filename = 0x7f5f94a6e1d8
    tmp_inc_filename = 0x0
    failure_retval = 0 '\000'

#8 0x00007f5fea6149e8 in execute_ex (execute_data=0x7f5ff050d0d0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_vm_execute.h:363

    ret = &lt;optimized out>
    original_in_execution = 0 '\000'

#9 0x00007f5fea5dab59 in dtrace_execute_ex (execute_data=<optimized out>) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_dtrace.c:73

    lineno = &lt;optimized out>
    scope = 0x0
    filename = &lt;optimized out>
    funcname = &lt;optimized out>
    classname = &lt;optimized out>

#10 0x00007f5fea5ec5e0 in zend_execute_scripts (type=type@entry=2, retval=retval@entry=0x0, file_count=file_count@entry=1) at /build/buildd/php5-5.5.9+dfsg/Zend/zend.c:1316

    files = {{gp_offset = 32, fp_offset = 32607, overflow_arg_area = 0x7fff4261b570, reg_save_area = 0x7fff4261b500}}
    i = 0
    file_handle = 0x7fff4261b5a0
    orig_op_array = 0x0
    orig_retval_ptr_ptr = 0x0
    orig_interactive = 0

#11 0x00007f5fea69c4ed in php_handler (r=<optimized out>) at /build/buildd/php5-5.5.9+dfsg/sapi/apache2handler/sapi_apache2.c:669

    zfd = {type = ZEND_HANDLE_FILENAME, filename = 0x7f5fee9a1800 "/var/www/vhosts/delivery/current/public/index.php", opened_path = 0x0, handle = {fd = 1113700060, fp = 0x7fff4261b6dc, stream = {handle = 0x7fff4261b6dc, isatty = -291891200, mmap = {
            len = 140049901037808, pos = 140050001698360, map = 0x7f5f00000000, buf = 0x7f5feea85bc8 "", old_handle = 0x7f5feea91748, old_closer = 0x7f5feea91748}, reader = 0x7f5fee9a1d00, fsizer = 0x7f5fee9a5f00, closer = 0x7f5fee9df290}}, free_filename = 0 '\000'}
    __orig_bailout = 0x0
    __bailout = {{__jmpbuf = {140050001714944, -7792186829664746521, 140050001714944, 140050006232960, 0, 140050001949328, 7791811881353544679, 7882218056389297127}, __mask_was_saved = 0, __saved_mask = {__val = {140050003104728, 140050002752280, 140050001692232, 1,
            140050002752280, 140050001692240, 140050001765624, 140050006232960, 0, 140050001949328, 140049949641891, 140050001696817, 2, 140050001692240, 140050001714944, 140050002624072}}}}
    ctx = 0x7f5fee9be690
    conf = &lt;optimized out>
    brigade = 0x7f5fee9c2c50
    bucket = &lt;optimized out>
    rv = &lt;optimized out>
    parent_req = 0x7f5fee9d3fb8

#12 0x00007f5feebb0680 in ap_run_handler (r=0x7f5fee9a5f00) at config.c:169

    pHook = 0x7f5feea83e70

---Type <return> to continue, or q <return> to quit---
n = 2
rv = 124
#13 0x00007f5feebb0bc9 in ap_invoke_handler (r=r@entry=0x7f5fee9a5f00) at config.c:439

    handler = &lt;optimized out>
    p = &lt;optimized out>
    result = &lt;optimized out>
    old_handler = 0x7f5feea9fbe8 "application/x-httpd-php"
    ignore = &lt;optimized out>

#14 0x00007f5feebc5c2c in ap_internal_redirect (new_uri=<optimized out>, r=<optimized out>) at http_request.c:644

    new = 0x7f5fee9a5f00
    access_status = &lt;optimized out>

#15 0x00007f5fe899acfc in handler_redirect (r=0x7f5fee9b24f8) at mod_rewrite.c:5063

No locals.
#16 0x00007f5feebb0680 in ap_run_handler (r=0x7f5fee9b24f8) at config.c:169

    pHook = 0x7f5feea83e98
    n = 3
    rv = 124

#17 0x00007f5feebb0bc9 in ap_invoke_handler (r=r@entry=0x7f5fee9b24f8) at config.c:439

    handler = &lt;optimized out>
    p = &lt;optimized out>
    result = &lt;optimized out>
    old_handler = 0x7f5fe89a31e9 "redirect-handler"
    ignore = &lt;optimized out>

#18 0x00007f5feebc5c2c in ap_internal_redirect (new_uri=<optimized out>, r=<optimized out>) at http_request.c:644

    new = 0x7f5fee9b24f8
    access_status = &lt;optimized out>

#19 0x00007f5fe899acfc in handler_redirect (r=0x7f5fee9b50a0) at mod_rewrite.c:5063

No locals.
#20 0x00007f5feebb0680 in ap_run_handler (r=0x7f5fee9b50a0) at config.c:169

    pHook = 0x7f5feea83e98
    n = 3
    rv = 124

#21 0x00007f5feebb0bc9 in ap_invoke_handler (r=r@entry=0x7f5fee9b50a0) at config.c:439

    handler = &lt;optimized out>
    p = &lt;optimized out>
    result = &lt;optimized out>
    old_handler = 0x7f5fe89a31e9 "redirect-handler"
    ignore = &lt;optimized out>

#22 0x00007f5feebc616a in ap_process_async_request (r=0x7f5fee9b50a0) at http_request.c:317

    access_status = 0

#23 0x00007f5feebc6444 in ap_process_request (r=r@entry=0x7f5fee9b50a0) at http_request.c:363

    bb = &lt;optimized out>
    b = &lt;optimized out>
    c = 0x7f5fee9df290
    rv = &lt;optimized out>

#24 0x00007f5feebc2f02 in ap_process_http_sync_connection (c=0x7f5fee9df290) at http_core.c:190

    r = 0x7f5fee9b50a0
    cs = 0x0
    csd = 0x7f5fee9df0a0
    mpm_state = 1

#25 ap_process_http_connection (c=0x7f5fee9df290) at http_core.c:231

No locals.
#26 0x00007f5feebb9cc0 in ap_run_process_connection (c=0x7f5fee9df290) at connection.c:41

    pHook = 0x7f5feea7c440
    n = 0
    rv = 124

#27 0x00007f5feebba0a8 in ap_process_connection (c=c@entry=0x7f5fee9df290, csd=<optimized out>) at connection.c:202

    rc = &lt;optimized out>

---Type <return> to continue, or q <return> to quit---
#28 0x00007f5feafbc767 in child_main (child_num_arg=child_num_arg@entry=197) at prefork.c:704

    current_conn = 0x7f5fee9df290
    csd = 0x7f5fee9df0a0
    thd = 0x7f5fee9e10a0
    osthd = 140050003421056
    ptrans = 0x7f5fee9df028
    allocator = 0x7f5ff0e9c390
    status = &lt;optimized out>
    i = &lt;optimized out>
    lr = &lt;optimized out>
    pollset = 0x7f5fee9e1158
    sbh = 0x7f5fee9e1150
    bucket_alloc = 0x7f5fee9db028
    last_poll_idx = 1
    lockfile = &lt;optimized out>

#29 0x00007f5feafbc9a6 in make_child (s=0x7f5feeb1ede0, slot=197) at prefork.c:800

    pid = 0

#30 0x00007f5feafbd60e in perform_idle_server_maintenance (p=<optimized out>) at prefork.c:902

    i = &lt;optimized out>
    idle_count = &lt;optimized out>
    ws = &lt;optimized out>
    free_length = &lt;optimized out>
    free_slots = {189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220}
    last_non_dead = &lt;optimized out>
    total_non_dead = &lt;optimized out>

#31 prefork_run (_pconf=<optimized out>, plog=<optimized out>, s=<optimized out>) at prefork.c:1090

    status = 32607
    pid = {pid = -1, in = 0x7f5feebd1048, out = 0xa, err = 0x7f5fee2afff6}
    child_slot = &lt;optimized out>
    exitwhy = (APR_PROC_SIGNAL | APR_PROC_SIGNAL_CORE | unknown: 3995795440)
    processed_status = &lt;optimized out>
    index = &lt;optimized out>
    remaining_children_to_start = 0
    rv = &lt;optimized out>

#32 0x00007f5feeb9769e in ap_run_mpm (pconf=0x7f5feeb4d028, plog=0x7f5feeb1a028, s=0x7f5feeb1ede0) at mpm_common.c:96

    pHook = 0x7f5feea7c978
    n = 0
    rv = 124

#33 0x00007f5feeb90e36 in main (argc=3, argv=0x7fff4261bde8) at main.c:777

    c = 0 '\000'
    showcompile = 0
    showdirectives = 0
    confname = 0x7f5feebd0607 "apache2.conf"
    def_server_root = 0x7f5feebd05fa "/etc/apache2"
    temp_error_log = 0x0
    error = &lt;optimized out>
    process = 0x7f5feeb4f118
    pconf = 0x7f5feeb4d028
    plog = 0x7f5feeb1a028
    ptemp = 0x7f5feeb1c028
    pcommands = 0x7f5feeb24028
    opt = 0x7f5feeb24118
    rv = &lt;optimized out>
    mod = 0x7f5feedf2160 &lt;ap_prelinked_modules+64>
    opt_arg = 0x7f5feeb4f028 "(0\265\356_\177"
    signal_server = &lt;optimized out>

[PHP Modules]
bcmath
bz2
calendar
Core
ctype
date
dba
dom
ereg
exif
FiftyOne_Degrees_Detector
fileinfo
filter
ftp
geoip
gettext
hash
iconv
intl
json
libxml
mbstring
mcrypt
memcache
memcached
mhash
mysql
mysqli
openssl
pcntl
pcre
PDO
pdo_mysql
phalcon
Phar
posix
readline
recode
Reflection
session
shmop
SimpleXML
soap
sockets
SPL
standard
sysvmsg
sysvsem
sysvshm
tokenizer
wddx
xml
xmlreader
xmlrpc
xmlwriter
xsl
Zend OPcache
zip
zlib

[Zend Modules]
Zend OPcache

Apache Compiled in modules:
core.c
mod_so.c
mod_watchdog.c
http_core.c
mod_log_config.c
mod_logio.c
mod_version.c
mod_unixd.c

Want to back this issue? Post a bounty on it! We accept bounties via Bountysource.
@baad777
Copy link
Author

baad777 commented Nov 6, 2014

Can someone help in some way?

@oh-ren
Copy link

oh-ren commented Nov 6, 2014

I am certainly no expert in this regard, but I can try to do some suggestions of course. Anyhow, you didn't mention the Phalcon ver. you're running? Are you running with the latest version, i.e. 1.3.4?
Upgrading PHP to the latest 5.5.x might help?
There are a lot of reported vague issues related to segmentation faults. It seems for certain issues it helps to disable OPcache. Not ideal I reckon, but it might help.
(I'd first try to ensure you're running the latest version(s). I also remember someone btw who moved back to 1.2.6 because of 'weird' issues with 1.3.x..) (of course that's not a solution, because there's not really a 1.2.x branch, 1.3.0 is the successor of 1.2.6)

@baad777
Copy link
Author

baad777 commented Nov 6, 2014

Phalcon version is 1.3.3, but we tried all versions between 1.2.6 and 1.3.4. Also we already tried disabling OPcache on all Phalcon versions with no luck. Issue still happen.

@oh-ren
Copy link

oh-ren commented Nov 6, 2014

Is it possible for you to upgrade PHP?

@baad777
Copy link
Author

baad777 commented Nov 6, 2014

We already tried with 5.5.9 and 5.5.17 and of course all Phalcon versions

@oh-ren
Copy link

oh-ren commented Nov 6, 2014

Seeing reports on stuff breaking at high volume, and other reports on seemingly random segfaults, do scare me a bit (much).

@dompie
Copy link

dompie commented Nov 6, 2014

Let me go on with suggestions:
The backtrace says at first (exit) point:

#0 0x00007f5fea6128c6 in zend_objects_store_put (object=0x7f5ff052e160, dtor=0x7f5fea60cb00 <zend_objects_destroy_object>, free_storage=0x7f5fe28830f0 <phalcon_config_object_dtor>, clone=0x0) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c:114
handle = 124

And line 114 said:

"114 /build/buildd/php5-5.5.9+dfsg/Zend/zend_objects_API.c: No such file or directory."

Maybe there have been some .c files removed after compilation which are still required?
Maybe some (op) cache files have been removed from outside php and php still thinks, they are present (or vice versa)?
Maybe a directory could not be created, because disk runs full in some rare cases?

@dugwood
Copy link
Contributor

dugwood commented Nov 13, 2014

@dompie: this isn't the issue, the gdb tries to load the file where the issue happens. Also you never run .c/.h/.zep files, which are not compiled, so not runnable.

I'm experiencing some segmentation fault too, the last one in phalcon 2: #3026 (comment)

For now, I've managed to go around each bug, but that's difficult to go forward with some of these tricky bugs.

@renskii: I know some high traffic websites that run phalcon 1.3.4 without any issue. Maybe the issue is the webserver, in your cases, or in mine. PHP wasn't segfaulting when I was running the CLI version, when the same script with the CGI version was segfaulting (I use lighttpd+fastcgi).

@bancuadrian: can you try the same thing under nginx/php-fpm or something like that? Maybe that would help. Another hint: https://bugs.php.net/bug.php?id=67089

@denniscuijpers
Copy link

@bancuadrian Don't know if you still have issues with php segfaulting. I had exactly the same backtraces and finally found a solution by first disabling php opcache to get a better stacktrace, turns out it has nothing to do with the script itself.

The problem is caused by apache not correctly handing multiple requests / http pipelining through a single connection. Turning KeepAlive off did the trick for me. See https://bugs.php.net/bug.php?id=68486 and https://issues.apache.org/bugzilla/show_bug.cgi?id=56984

Hope this helps!

@scresp0
Copy link

scresp0 commented Mar 27, 2015

Hi,

I'm running Ubuntu with nginx + wordpress +php5-fpm and I'm having this issue just with some posts or web pages on my installation..

..and the most amazing thing is, that it doesn't happen fetching these URLs from a mobile device (it uses a mobile theme then).. Debugging what's going on:

For instance, I access this URL from my computer web browser and the php5 execution crashes, so the reader gets a 502 error.

http://sophosiberia.es/5-sencillos-consejos-para-evitar-ransomware/
or
http://sophosiberia.es/boletin-seguridad/

If you open any of these two URLs from a mobile phone, they just work!

So taking a look at /var/log/apport.log I can just read:
ERROR: apport (pid 8659) Fri Mar 27 18:00:11 2015: called for pid 7445, signal 11, core limit 0
ERROR: apport (pid 8659) Fri Mar 27 18:00:11 2015: executable: /usr/sbin/php5-fpm (command line "php-fpm:\ pool\ www")
ERROR: apport (pid 8659) Fri Mar 27 18:00:11 2015: is_closing_session(): no DBUS_SESSION_BUS_ADDRESS in environment
ERROR: apport (pid 8659) Fri Mar 27 18:00:37 2015: wrote report /var/crash/_usr_sbin_php5-fpm.33.crash

So, let's go: Let's have a look at that crash file:

cd /var/crash
apport-cli ./_usr_sbin_php5-fpm.33.crash

*** Send problem report to the developers?

After the problem report has been sent, please fill out the form in the
automatically opened web browser.

What would you like to do? Your options are:
S: Send report (10.4 MB)
E: Examine locally
V: View report
K: Keep report file for sending later or copying to somewhere else
I: Cancel and ignore future crashes of this program version
C: Cancel
Please choose (S/E/V/K/I/C): E
*** Collecting problem information

The collected information can be sent to the developers to improve the
application. This might take a few minutes.
......................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................

and so.. takes a while, and at the end:

For help, type "help".
Type "apropos word" to search for commands related to "word".
Reading symbols from /tmp/apport_sandbox_IvosRM//usr/sbin/php5-fpm...Reading symbols from /tmp/apport_sandbox_IvosRM/usr/lib/debug/usr/sbin/php5-fpm...done.
done.
[New LWP 7450]
warning: Could not load shared library symbols for /usr/lib/php5/20121212/newrelic.so.
Do you need "set solib-search-path" or "set sysroot"?
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
warning: the debug information found in "/tmp/apport_sandbox_IvosRM/usr/lib/debug/usr/lib/php5/20121212/mysql.so" does not match "/tmp/apport_sandbox_IvosRM/usr/lib/php5/20121212/mysql.so" (CRC mismatch).

warning: the debug information found in "/tmp/apport_sandbox_IvosRM/usr/lib/debug/usr/lib/php5/20121212/mysqli.so" does not match "/tmp/apport_sandbox_IvosRM/usr/lib/php5/20121212/mysqli.so" (CRC mismatch).

warning: the debug information found in "/tmp/apport_sandbox_IvosRM/usr/lib/debug/usr/lib/php5/20121212/pdo_mysql.so" does not match "/tmp/apport_sandbox_IvosRM/usr/lib/php5/20121212/pdo_mysql.so" (CRC mismatch).

Core was generated by `php-fpm: pool www '.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x00000000006efb64 in zend_get_class_entry (zobject=0x334fb48) at /build/buildd/php5-5.5.9+dfsg/Zend/zend_API.c:237
237 /build/buildd/php5-5.5.9+dfsg/Zend/zend_API.c: No such file or directory.

Same thing commented here.. But I have no idea what it means, and why it happens. Any ideas?

thanks in advance!

@andresgutierrez
Copy link
Contributor

Could you please try again using Phalcon 2 or close this issue as 1.3.x is no longer supported

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
7 participants
@andresgutierrez @denniscuijpers @dompie @oh-ren @dugwood @baad777 @scresp0