From 38960e5f004074b70f789ab244fb519938b8017a Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Thu, 12 Sep 2024 17:48:58 +0000 Subject: [PATCH 1/2] Avoid undefined behavior when subtracting tagged pointers When compiling with HardwareAddressSanitizer, the top byte of pointers are tagged, hence subtracting two pointers may overflow. Overflow is undefined behavior for signed integers, resulting in errors such as: third_party/nanobind/src/nb_type.cpp:123:50: runtime error: signed integer overflow: -6701078944169102656 - 9079534012674239392 cannot be represented in type 'intptr_t' (aka 'long') This patch fixes the issue by using unsigned integers, for which overflow is well-defined. --- src/nb_type.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nb_type.cpp b/src/nb_type.cpp index 23ff4e3f..11e1eeed 100644 --- a/src/nb_type.cpp +++ b/src/nb_type.cpp @@ -124,7 +124,10 @@ PyObject *inst_new_ext(PyTypeObject *tp, void *value) { } // Compute offset to instance value - int32_t offset = (int32_t) ((intptr_t) value - (intptr_t) self); + // Use uint64_t because subtracting tagged pointers (e.g., with + // HardwareAddressSanitizer) may overflow, which is undefined behavior for + // signed integers. + int32_t offset = (int32_t) ((uint64_t) value - (uint64_t) self); bool direct = (intptr_t) self + offset == (intptr_t) value; if (NB_UNLIKELY(!direct)) { From 30b8826d42a3d6b1ce6271c22a53da084a52b541 Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Thu, 12 Sep 2024 19:20:56 +0000 Subject: [PATCH 2/2] Use uintptr_t, as suggested by phawkins --- src/nb_type.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nb_type.cpp b/src/nb_type.cpp index 11e1eeed..b86a38d0 100644 --- a/src/nb_type.cpp +++ b/src/nb_type.cpp @@ -127,7 +127,7 @@ PyObject *inst_new_ext(PyTypeObject *tp, void *value) { // Use uint64_t because subtracting tagged pointers (e.g., with // HardwareAddressSanitizer) may overflow, which is undefined behavior for // signed integers. - int32_t offset = (int32_t) ((uint64_t) value - (uint64_t) self); + int32_t offset = (int32_t) ((uintptr_t) value - (uintptr_t) self); bool direct = (intptr_t) self + offset == (intptr_t) value; if (NB_UNLIKELY(!direct)) {