Segfault with vector of vectors.

[gsolberg]

Possible duplicate of "segfault with large data" #71

When converting a relatively large python list of lists into a Vec<Vec<Vec<f64>>> I sometimes get a segfault. When using the rust address sanitizer I consistently get a heap-use-after-free error unless the list is small. (A small list gets a clean run with the address sanitizer.) I get identical results on OS/X 10.11.6 + Python 3.6.4 and Ubuntu 18.04 + Python 3.6.5. I've been chasing this bug for a while across a few rust versions and am now using rustc 1.29.0-nightly (6a1c0637c 2018-07-23).

Cargo.toml

[package]
name = "pytest"
version = "0.1.0"

[dependencies]
pyo3 = "0.3.2"

src/main.rs

#![feature(use_extern_macros, specialization)]
extern crate pyo3;
use pyo3::prelude::*;

// range(2) seems to work while range(20) or larger generates heap-use-after-free error
const SCRIPT: &str = r#"
vector = [[[float(x) for x in range(20)] for _ in range(20)] for _ in range(20)]
vec_test.test(vector)
"#;

fn main() {
    let gil = Python::acquire_gil();
    let py = gil.python();
    let vec_test = py.init_ref(|token| VecTest { token }).unwrap();
    let locals = PyDict::new(py);
    locals.set_item("vec_test", vec_test).unwrap();
    py.run(SCRIPT, None, Some(locals)).unwrap();
}

#[pyclass]
struct VecTest {
    token: PyToken,
}

#[pymethods]
impl VecTest {
    fn test(&self, vector: Vec<Vec<Vec<f64>>>) -> PyResult<()> {
        println!("vector {:?}", vector);
        Ok(())
    }
}

Address sanitizer output

$ RUSTFLAGS="-Z sanitizer=address" cargo run --target x86_64-apple-darwin
   Compiling pytest v0.1.0 (file:///Users/gsolberg/pytest)
    Finished dev [unoptimized + debuginfo] target(s) in 2.18s
     Running `target/x86_64-apple-darwin/debug/pytest`
=================================================================
==60851==ERROR: AddressSanitizer: heap-use-after-free on address 0x61d000039898 at pc 0x00010ca9c546 bp 0x7fff5326a990 sp 0x7fff5326a988
READ of size 8 at 0x61d000039898 thread T0
    #0 0x10ca9c545 in _$LT$pyo3..object..PyObject$u20$as$u20$pyo3..python..ToPyPointer$GT$::as_ptr::h791cbf395574836b object.rs:293
    #1 0x10caa59c4 in _$LT$pyo3..objects..PyObjectRef$u20$as$u20$pyo3..python..ToPyPointer$GT$::as_ptr::h3481796642241901 mod.rs:80
    #2 0x10ca44a22 in _$LT$pyo3..objects..iterator..PyIterator$LT$$u27$p$GT$$u20$as$u20$core..iter..iterator..Iterator$GT$::next::h6ab3b139b10480d4 iterator.rs:48
    #3 0x10c9b8214 in pyo3::objects::sequence::extract_sequence::hb776fb05595d2580 sequence.rs:242
    #4 0x10c9a8c5b in pyo3::objects::sequence::_$LT$impl$u20$pyo3..conversion..FromPyObject$LT$$u27$a$GT$$u20$for$u20$alloc..vec..Vec$LT$T$GT$$GT$::extract::hf732d1bf0a69066c sequence.rs:215
    #5 0x10c9dfd83 in _$LT$T$u20$as$u20$pyo3..objectprotocol..ObjectProtocol$GT$::extract::h9d06a7278a126c24 objectprotocol.rs:451
    #6 0x10c9cfa25 in _$LT$pytest..VecTest$u20$as$u20$pyo3..class..methods..PyMethodsProtocolImpl$GT$::py_methods::METHODS::__wrap::h3178d952aa0a77cf main.rs:24
    #7 0x10d94149a in _PyCFunction_FastCallDict (Python:x86_64+0x4449a)
    #8 0x10d9a8e5b in call_function (Python:x86_64+0xabe5b)
    #9 0x10d9a6266 in _PyEval_EvalFrameDefault (Python:x86_64+0xa9266)
    #10 0x10d9a9638 in _PyEval_EvalCodeWithName (Python:x86_64+0xac638)
    #11 0x10d99f8b9 in PyEval_EvalCode (Python:x86_64+0xa28b9)
    #12 0x10d9c9d54 in run_mod (Python:x86_64+0xccd54)
    #13 0x10d9c8ebe in PyRun_StringFlags (Python:x86_64+0xcbebe)
    #14 0x10ca4f271 in pyo3::python::Python::run_code::ha1f38faf716a0f77 python.rs:160
    #15 0x10ca4e1ac in pyo3::python::Python::run::hc2823a60410d4d8a python.rs:136
    #16 0x10c9ce996 in pytest::main::h2aeb7d1c888f9b6d main.rs:16
    #17 0x10c9e28ed in std::rt::lang_start::_$u7b$$u7b$closure$u7d$$u7d$::h5eb754103ee7354f rt.rs:74
    #18 0x10cab77f7 in std::panicking::try::do_call::h90c8830c237e1486 panicking.rs:310
    #19 0x10cac3d8e in __rust_maybe_catch_panic lib.rs:106
    #20 0x10caaf7fc in std::rt::lang_start_internal::h15876038f4767761 panicking.rs:289
    #21 0x10c9e2860 in std::rt::lang_start::hef62cdce06921e64 rt.rs:74
    #22 0x10c9cfe11 in main (pytest:x86_64+0x10003de11)
    #23 0x7fff995215ac in start (libdyld.dylib:x86_64+0x35ac)

0x61d000039898 is located 24 bytes inside of 2048-byte region [0x61d000039880,0x61d00003a080)
freed by thread T0 here:
    #0 0x10cc35dc7 in wrap_realloc (libclang_rt.asan_osx_dynamic.dylib:x86_64+0x59dc7)
    #1 0x10ca3f9fe in alloc::alloc::realloc::h033464698394b9f8 alloc.rs:98
    #2 0x10ca3f05c in _$LT$alloc..alloc..Global$u20$as$u20$core..alloc..Alloc$GT$::realloc::h3cb2b509bb0f9e74 alloc.rs:138
    #3 0x10ca6ab01 in _$LT$alloc..raw_vec..RawVec$LT$T$C$$u20$A$GT$$GT$::reserve_internal::h88ebb3aedec4390a raw_vec.rs:677
    #4 0x10ca6bfe8 in _$LT$alloc..raw_vec..RawVec$LT$T$C$$u20$A$GT$$GT$::reserve::h63a55845084a7690 raw_vec.rs:502
    #5 0x10ca8487b in _$LT$alloc..vec..Vec$LT$T$GT$$GT$::reserve::he1efa122a50b01ba vec.rs:464
    #6 0x10ca84565 in _$LT$alloc..vec..Vec$LT$T$GT$$GT$::push::h6f7f7aa981b5f21f vec.rs:1066
    #7 0x10ca702cd in pyo3::pythonrun::register_borrowed::h1c09df75d121a42f pythonrun.rs:247
    #8 0x10ca4fc74 in pyo3::python::Python::from_borrowed_ptr::h17bed924eb2253a9 python.rs:353
    #9 0x10c9ebd60 in pyo3::objects::iterator::PyIterator::from_object::hf842d63b7462efd3 iterator.rs:28
    #10 0x10c9b9234 in _$LT$T$u20$as$u20$pyo3..objectprotocol..ObjectProtocol$GT$::iter::h0fb5f8dcaf6f9acd objectprotocol.rs:421
    #11 0x10c9b6553 in pyo3::objects::sequence::extract_sequence::h8f61111a3c0798e2 sequence.rs:242
    #12 0x10c9a8c2b in pyo3::objects::sequence::_$LT$impl$u20$pyo3..conversion..FromPyObject$LT$$u27$a$GT$$u20$for$u20$alloc..vec..Vec$LT$T$GT$$GT$::extract::h29944a818d1d8cbc sequence.rs:215
    #13 0x10c9dfd43 in _$LT$T$u20$as$u20$pyo3..objectprotocol..ObjectProtocol$GT$::extract::h00c022a169a8281a objectprotocol.rs:451
    #14 0x10c9b8527 in pyo3::objects::sequence::extract_sequence::hb776fb05595d2580 sequence.rs:243
    #15 0x10c9a8c5b in pyo3::objects::sequence::_$LT$impl$u20$pyo3..conversion..FromPyObject$LT$$u27$a$GT$$u20$for$u20$alloc..vec..Vec$LT$T$GT$$GT$::extract::hf732d1bf0a69066c sequence.rs:215
    #16 0x10c9dfd83 in _$LT$T$u20$as$u20$pyo3..objectprotocol..ObjectProtocol$GT$::extract::h9d06a7278a126c24 objectprotocol.rs:451
    #17 0x10c9cfa25 in _$LT$pytest..VecTest$u20$as$u20$pyo3..class..methods..PyMethodsProtocolImpl$GT$::py_methods::METHODS::__wrap::h3178d952aa0a77cf main.rs:24
    #18 0x10d94149a in _PyCFunction_FastCallDict (Python:x86_64+0x4449a)
    #19 0x10d9a8e5b in call_function (Python:x86_64+0xabe5b)
    #20 0x10d9a6266 in _PyEval_EvalFrameDefault (Python:x86_64+0xa9266)
    #21 0x10d9a9638 in _PyEval_EvalCodeWithName (Python:x86_64+0xac638)
    #22 0x10d99f8b9 in PyEval_EvalCode (Python:x86_64+0xa28b9)
    #23 0x10d9c9d54 in run_mod (Python:x86_64+0xccd54)
    #24 0x10d9c8ebe in PyRun_StringFlags (Python:x86_64+0xcbebe)
    #25 0x10ca4f271 in pyo3::python::Python::run_code::ha1f38faf716a0f77 python.rs:160
    #26 0x10ca4e1ac in pyo3::python::Python::run::hc2823a60410d4d8a python.rs:136
    #27 0x10c9ce996 in pytest::main::h2aeb7d1c888f9b6d main.rs:16
    #28 0x10c9e28ed in std::rt::lang_start::_$u7b$$u7b$closure$u7d$$u7d$::h5eb754103ee7354f rt.rs:74
    #29 0x10cab77f7 in std::panicking::try::do_call::h90c8830c237e1486 panicking.rs:310

previously allocated by thread T0 here:
    #0 0x10cc35a43 in wrap_malloc (libclang_rt.asan_osx_dynamic.dylib:x86_64+0x59a43)
    #1 0x10ca3f681 in alloc::alloc::alloc::had80aef684a14c70 alloc.rs:62
    #2 0x10ca3ebea in _$LT$alloc..alloc..Global$u20$as$u20$core..alloc..Alloc$GT$::alloc::hfaa420302d241353 alloc.rs:123
    #3 0x10ca659fd in _$LT$alloc..raw_vec..RawVec$LT$T$C$$u20$A$GT$$GT$::allocate_in::h3e36b21d82eb489b raw_vec.rs:106
    #4 0x10ca654c6 in _$LT$alloc..raw_vec..RawVec$LT$T$GT$$GT$::with_capacity::h308b306c945afde9 raw_vec.rs:150
    #5 0x10ca838e2 in _$LT$alloc..vec..Vec$LT$T$GT$$GT$::with_capacity::ha67ea34b4972a9c0 vec.rs:362
    #6 0x10ca6e462 in pyo3::pythonrun::ReleasePool::new::h5b4752ef86da806f pythonrun.rs:124
    #7 0x10ca6dff0 in pyo3::pythonrun::init_once::_$u7b$$u7b$closure$u7d$$u7d$::habcda9bbc6ed676f pythonrun.rs:73
    #8 0x10ca53c43 in std::sync::once::Once::call_once::_$u7b$$u7b$closure$u7d$$u7d$::h3307e3a12bacfb31 once.rs:232
    #9 0x10cab613c in std::sync::once::Once::call_inner::h5beb36057172bda5 once.rs:349
    #10 0x10ca53a26 in std::sync::once::Once::call_once::hec4ab66b57de294b once.rs:232
    #11 0x10ca6df26 in pyo3::pythonrun::init_once::hfda49232b4d2fcff pythonrun.rs:71
    #12 0x10ca6de6c in pyo3::pythonrun::prepare_freethreaded_python::_$u7b$$u7b$closure$u7d$$u7d$::hcd6f0860bc50a886 pythonrun.rs:64
    #13 0x10ca53cb3 in std::sync::once::Once::call_once::_$u7b$$u7b$closure$u7d$$u7d$::h81e3cc1ce71598bd once.rs:232
    #14 0x10cab613c in std::sync::once::Once::call_inner::h5beb36057172bda5 once.rs:349
    #15 0x10ca53566 in std::sync::once::Once::call_once::h756144e04b111149 once.rs:232
    #16 0x10ca6d0b6 in pyo3::pythonrun::prepare_freethreaded_python::hcd78fcb27c7c7dff pythonrun.rs:38
    #17 0x10ca703f7 in pyo3::pythonrun::GILGuard::acquire::h9314da82cad342fd pythonrun.rs:257
    #18 0x10c9ba1d3 in pyo3::python::Python::acquire_gil::h252c3be9648755da python.rs:106
    #19 0x10c9ce6dc in pytest::main::h2aeb7d1c888f9b6d main.rs:11
    #20 0x10c9e28ed in std::rt::lang_start::_$u7b$$u7b$closure$u7d$$u7d$::h5eb754103ee7354f rt.rs:74
    #21 0x10cab77f7 in std::panicking::try::do_call::h90c8830c237e1486 panicking.rs:310
    #22 0x10cac3d8e in __rust_maybe_catch_panic lib.rs:106
    #23 0x10caaf7fc in std::rt::lang_start_internal::h15876038f4767761 panicking.rs:289
    #24 0x10c9e2860 in std::rt::lang_start::hef62cdce06921e64 rt.rs:74
    #25 0x10c9cfe11 in main (pytest:x86_64+0x10003de11)
    #26 0x7fff995215ac in start (libdyld.dylib:x86_64+0x35ac)

SUMMARY: AddressSanitizer: heap-use-after-free object.rs:293 in _$LT$pyo3..object..PyObject$u20$as$u20$pyo3..python..ToPyPointer$GT$::as_ptr::h791cbf395574836b
Shadow bytes around the buggy address:
  0x1c3a000072c0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x1c3a000072d0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x1c3a000072e0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x1c3a000072f0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x1c3a00007300: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
=>0x1c3a00007310: fd fd fd[fd]fd fd fd fd fd fd fd fd fd fd fd fd
  0x1c3a00007320: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x1c3a00007330: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x1c3a00007340: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x1c3a00007350: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x1c3a00007360: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
Shadow byte legend (one shadow byte represents 8 application bytes):
  Addressable:           00
  Partially addressable: 01 02 03 04 05 06 07
  Heap left redzone:       fa
  Freed heap region:       fd
  Stack left redzone:      f1
  Stack mid redzone:       f2
  Stack right redzone:     f3
  Stack after return:      f5
  Stack use after scope:   f8
  Global redzone:          f9
  Global init order:       f6
  Poisoned by user:        f7
  Container overflow:      fc
  Array cookie:            ac
  Intra object redzone:    bb
  ASan internal:           fe
  Left alloca redzone:     ca
  Right alloca redzone:    cb
  Shadow gap:              cc
==60851==ABORTING
Abort trap: 6

[kngwyu]

I can't reproduce this in 0.5.0-alpha.1.
Does anyone know what fixed this?

[fdoyon]

@kngwyu - are you testing on Linux?

This looks like the same kind of issue as #159 and the tentative PR #268 could be a proper fix as the PyListIterator::next method does also create a GIL before borrowing. I will try to repro on my machine but this seems eerily similar.

[konstin]

I think this is fixed in 0.5.1, if not please open another issue

[Shnatsel]

Use-after-free is an exploitable security vulnerability. Please add this issue to the Rust security advisory database so people could check if they're running a vulnerable version and upgrade.