Created
March 27, 2024 13:05
-
-
Save thrix/e38a80a2f1c6321b2d6a68fb6cdb3a3e to your computer and use it in GitHub Desktop.
Python debugging with GDB on Fedora
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| From https://github.com/teemtee/tmt/issues/2501 | |
| I can reproduce the issue with https://github.com/teemtee/tmt/issues/2501#issuecomment-1954532963 reproducer. | |
| ``` | |
| [root@e67b66cee0cb /]# dnf install -y python3-debug | |
| [root@e67b66cee0cb /]# cd | |
| [root@e67b66cee0cb ~]# curl -O https://raw.githubusercontent.com/python/cpython/3.12/Tools/gdb/libpython.py | |
| [root@e67b66cee0cb ~]# export DEBUGINFOD_URLS=https://debuginfod.fedoraproject.org/ | |
| [root@e67b66cee0cb ~]# gdb -args python3-debug /usr/bin/tmt run prov -h beaker --image rhel-9999 | |
| (gdb) set debuginfod enabled on | |
| (gdb) source libpython.py | |
| (gdb) run | |
| (...) | |
| python3-debug: /builddir/build/BUILD/Python-3.12.2/Include/cpython/tupleobject.h:23: PyTuple_GET_SIZE: Assertion `PyTuple_Check(op)' failed. | |
| (gdb) where | |
| (gdb) where | |
| #0 __pthread_kill_implementation (threadid=<optimized out>, signo=signo@entry=6, no_tid=no_tid@entry=0) at pthread_kill.c:44 | |
| #1 0x00007f22215288a3 in __pthread_kill_internal (signo=6, threadid=<optimized out>) at pthread_kill.c:78 | |
| #2 0x00007f22214d68ee in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26 | |
| #3 0x00007f22214be8ff in __GI_abort () at abort.c:79 | |
| #4 0x00007f22214be81b in __assert_fail_base (fmt=0x7f222163daf8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", | |
| assertion=assertion@entry=0x7f2221b80176 "PyTuple_Check(op)", file=file@entry=0x7f2221c844f8 "/builddir/build/BUILD/Python-3.12.2/Include/cpython/tupleobject.h", | |
| line=line@entry=23, function=function@entry=0x7f2221b9deb0 <__PRETTY_FUNCTION__.12.lto_priv.7> "PyTuple_GET_SIZE") at assert.c:92 | |
| #5 0x00007f22214cec57 in __assert_fail (assertion=0x7f2221b80176 "PyTuple_Check(op)", | |
| file=0x7f2221c844f8 "/builddir/build/BUILD/Python-3.12.2/Include/cpython/tupleobject.h", line=23, | |
| function=0x7f2221b9deb0 <__PRETTY_FUNCTION__.12.lto_priv.7> "PyTuple_GET_SIZE") at assert.c:101 | |
| #6 0x00007f22218780d2 in PyTuple_GET_SIZE (op={}) at /usr/src/debug/python3.12-3.12.2-2.fc39.x86_64/Include/cpython/tupleobject.h:23 | |
| #7 0x00007f2221878d1d in method_vectorcall (method=<method at remote 0x7f221ce90410>, args=0x7f221d9d85a8, nargsf=2, kwnames={}) | |
| at /usr/src/debug/python3.12-3.12.2-2.fc39.x86_64/Objects/classobject.c:66 | |
| #8 0x00007f221ccb44b7 in __Pyx_Py3MetaclassPrepare (metaclass=metaclass@entry=<type at remote 0x7f22100df720>, | |
| bases=bases@entry=(<EnumType(_generate_next_value_=<staticmethod at remote 0x7f222103b890>, __module__='enum', __doc__='\n Enum where members are also (and must be) ints\n ', _new_member_=<built-in method __new__ of type object at remote 0x7f2221dced00>, _use_args_=True, _member_names_=[], _member_map_={}, _value2member_map_={}, _unhashable_values_=[], _member_type_=<type at remote 0x7f2221dced00>, _value_repr_=<wrapper_descriptor at remote 0x7f222129ed50>, __dict__=<getset_descriptor at remote 0x7f2221054890>, __format__=<method_descriptor at remote 0x7f22212a01d0>, __str__=<wrapper_descriptor at remote 0x7f222129ed50>, __repr__=<function at remote 0x7f222104ec90>, __new__=<function at remote 0x7f222104e990>) at remote 0x55d5e80d9030>,), name='RequirementFlag', qualname='RequirementFlag', mkw=mkw@entry={}, | |
| modname='gssapi.raw.types', doc=0x0) at gssapi/raw/types.c:13392 | |
| #9 0x00007f221ccb58d5 in __pyx_pymod_exec_types (__pyx_pyinit_module=<optimized out>) at gssapi/raw/types.c:10426 | |
| (gdb) py-bt | |
| Traceback (most recent call first): | |
| <built-in method exec_dynamic of module object at remote 0x7f222130a390> | |
| File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed | |
| File "<frozen importlib._bootstrap_external>", line 1297, in exec_module | |
| File "<frozen importlib._bootstrap>", line 935, in _load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1360, in _find_and_load | |
| <built-in method exec_dynamic of module object at remote 0x7f222130a390> | |
| File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed | |
| File "<frozen importlib._bootstrap_external>", line 1297, in exec_module | |
| File "<frozen importlib._bootstrap>", line 935, in _load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1360, in _find_and_load | |
| <built-in method __import__ of module object at remote 0x7f22212f26f0> | |
| <built-in method exec_dynamic of module object at remote 0x7f222130a390> | |
| File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed | |
| File "<frozen importlib._bootstrap_external>", line 1297, in exec_module | |
| File "<frozen importlib._bootstrap>", line 935, in _load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1360, in _find_and_load | |
| <built-in method exec_dynamic of module object at remote 0x7f222130a390> | |
| File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed | |
| File "<frozen importlib._bootstrap_external>", line 1297, in exec_module | |
| File "<frozen importlib._bootstrap>", line 935, in _load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1360, in _find_and_load | |
| <built-in method __import__ of module object at remote 0x7f22212f26f0> | |
| <built-in method exec_dynamic of module object at remote 0x7f222130a390> | |
| File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed | |
| File "<frozen importlib._bootstrap_external>", line 1297, in exec_module | |
| File "<frozen importlib._bootstrap>", line 935, in _load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1360, in _find_and_load | |
| File "/usr/lib64/python3.12/site-packages/gssapi/raw/__init__.py", line 50, in <module> | |
| from gssapi.raw.creds import * # noqa | |
| (...) | |
| (gdb) frame 6 | |
| #6 0x00007fb1531fa0d2 in PyTuple_GET_SIZE (op={}) at /usr/src/debug/python3.12-3.12.2-2.fc39.x86_64/Include/cpython/tupleobject.h:23 | |
| 23 PyTupleObject *tuple = _PyTuple_CAST(op); | |
| (gdb) frame 7 | |
| #7 0x00007fb1531fad1d in method_vectorcall (method=<method at remote 0x7fb14e8b44d0>, args=0x7fb14fdd85a8, nargsf=2, kwnames={}) | |
| at /usr/src/debug/python3.12-3.12.2-2.fc39.x86_64/Objects/classobject.c:66 | |
| (gdb) list | |
| 61 result = _PyObject_VectorcallTstate(tstate, func, newargs, | |
| 62 nargs, kwnames); | |
| 63 newargs[0] = tmp; | |
| 64 } | |
| 65 else { | |
| 66 Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); <== HERE | |
| 67 Py_ssize_t totalargs = nargs + nkwargs; | |
| 68 if (totalargs == 0) { | |
| 69 return _PyObject_VectorcallTstate(tstate, func, &self, 1, NULL); | |
| 70 } | |
| (gdb) frame 8 | |
| #8 0x00007fb14e63b4b7 in __Pyx_Py3MetaclassPrepare (metaclass=metaclass@entry=<type at remote 0x7fb1480df6f0>, | |
| bases=bases@entry=(<EnumType(_generate_next_value_=<staticmethod at remote 0x7fb152b2a1c0>, __module__='enum', __doc__='\n Enum where members are also (and must be) ints\n ', _new_member_=<built-in method __new__ of type object at remote 0x7fb153750d00>, _use_args_=True, _member_names_=[], _member_map_={}, _value2member_map_={}, _unhashable_values_=[], _member_type_=<type at remote 0x7fb153750d00>, _value_repr_=<wrapper_descriptor at remote 0x7fb152c22d50>, __dict__=<getset_descriptor at remote 0x7fb1529d4890>, __format__=<method_descriptor at remote 0x7fb152c241d0>, __str__=<wrapper_descriptor at remote 0x7fb152c22d50>, __repr__=<function at remote 0x7fb1529cec90>, __new__=<function at remote 0x7fb1529ce990>) at remote 0x5563182d0030>,), name='RequirementFlag', qualname='RequirementFlag', mkw=mkw@entry={}, | |
| modname='gssapi.raw.types', doc=0x0) at gssapi/raw/types.c:13392 | |
| (gdb) list | |
| 13387 PyObject *ns; | |
| 13388 if (metaclass) { | |
| 13389 PyObject *prep = __Pyx_PyObject_GetAttrStrNoError(metaclass, __pyx_n_s_prepare); | |
| 13390 if (prep) { | |
| 13391 PyObject *pargs[3] = {NULL, name, bases}; | |
| 13392 ns = __Pyx_PyObject_FastCallDict(prep, pargs+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, mkw); <==== HERE === | |
| 13393 Py_DECREF(prep); | |
| 13394 } else { | |
| 13395 if (unlikely(PyErr_Occurred())) | |
| 13396 return NULL; | |
| ``` | |
| The bug occurs on `from gssapi.raw.creds import * # noqa`: when the C extension `gssapi` is imported, Cython creates types in the module exec function (`__pyx_pymod_exec_types()`). | |
| Problem: `__Pyx_Py3MetaclassPrepare()` pass a dictionary as *kwnames* to `method_vectorcall()`, whereas *kwnames* must be a tuple. Later, an assertion fails since we get the wrong type. | |
| __Pyx_Py3MetaclassPrepare() pass its `mkw` argument to `method_vectorcall()` as `kwnames`: | |
| * frame 9, __pyx_pymod_exec_types(): *__pyx_t_4* | |
| * frame 8, __Pyx_Py3MetaclassPrepare(): *mkw* | |
| * frame 7, method_vectorcall(): *kwnames* | |
| __pyx_pymod_exec_types() creates it as a dictionary with: | |
| ```c | |
| __pyx_t_4 = __Pyx_PyDict_NewPresized(0); | |
| I can reproduce the issue with https://github.com/teemtee/tmt/issues/2501#issuecomment-1954532963 reproducer. | |
| ``` | |
| [root@e67b66cee0cb /]# dnf install -y python3-debug | |
| [root@e67b66cee0cb /]# cd | |
| [root@e67b66cee0cb ~]# curl -O https://raw.githubusercontent.com/python/cpython/3.12/Tools/gdb/libpython.py | |
| [root@e67b66cee0cb ~]# export DEBUGINFOD_URLS=https://debuginfod.fedoraproject.org/ | |
| [root@e67b66cee0cb ~]# gdb -args python3-debug /usr/bin/tmt run prov -h beaker --image rhel-9999 | |
| (gdb) set debuginfod enabled on | |
| (gdb) source libpython.py | |
| (gdb) run | |
| (...) | |
| python3-debug: /builddir/build/BUILD/Python-3.12.2/Include/cpython/tupleobject.h:23: PyTuple_GET_SIZE: Assertion `PyTuple_Check(op)' failed. | |
| (gdb) where | |
| (gdb) where | |
| #0 __pthread_kill_implementation (threadid=<optimized out>, signo=signo@entry=6, no_tid=no_tid@entry=0) at pthread_kill.c:44 | |
| #1 0x00007f22215288a3 in __pthread_kill_internal (signo=6, threadid=<optimized out>) at pthread_kill.c:78 | |
| #2 0x00007f22214d68ee in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26 | |
| #3 0x00007f22214be8ff in __GI_abort () at abort.c:79 | |
| #4 0x00007f22214be81b in __assert_fail_base (fmt=0x7f222163daf8 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", | |
| assertion=assertion@entry=0x7f2221b80176 "PyTuple_Check(op)", file=file@entry=0x7f2221c844f8 "/builddir/build/BUILD/Python-3.12.2/Include/cpython/tupleobject.h", | |
| line=line@entry=23, function=function@entry=0x7f2221b9deb0 <__PRETTY_FUNCTION__.12.lto_priv.7> "PyTuple_GET_SIZE") at assert.c:92 | |
| #5 0x00007f22214cec57 in __assert_fail (assertion=0x7f2221b80176 "PyTuple_Check(op)", | |
| file=0x7f2221c844f8 "/builddir/build/BUILD/Python-3.12.2/Include/cpython/tupleobject.h", line=23, | |
| function=0x7f2221b9deb0 <__PRETTY_FUNCTION__.12.lto_priv.7> "PyTuple_GET_SIZE") at assert.c:101 | |
| #6 0x00007f22218780d2 in PyTuple_GET_SIZE (op={}) at /usr/src/debug/python3.12-3.12.2-2.fc39.x86_64/Include/cpython/tupleobject.h:23 | |
| #7 0x00007f2221878d1d in method_vectorcall (method=<method at remote 0x7f221ce90410>, args=0x7f221d9d85a8, nargsf=2, kwnames={}) | |
| at /usr/src/debug/python3.12-3.12.2-2.fc39.x86_64/Objects/classobject.c:66 | |
| #8 0x00007f221ccb44b7 in __Pyx_Py3MetaclassPrepare (metaclass=metaclass@entry=<type at remote 0x7f22100df720>, | |
| bases=bases@entry=(<EnumType(_generate_next_value_=<staticmethod at remote 0x7f222103b890>, __module__='enum', __doc__='\n Enum where members are also (and must be) ints\n ', _new_member_=<built-in method __new__ of type object at remote 0x7f2221dced00>, _use_args_=True, _member_names_=[], _member_map_={}, _value2member_map_={}, _unhashable_values_=[], _member_type_=<type at remote 0x7f2221dced00>, _value_repr_=<wrapper_descriptor at remote 0x7f222129ed50>, __dict__=<getset_descriptor at remote 0x7f2221054890>, __format__=<method_descriptor at remote 0x7f22212a01d0>, __str__=<wrapper_descriptor at remote 0x7f222129ed50>, __repr__=<function at remote 0x7f222104ec90>, __new__=<function at remote 0x7f222104e990>) at remote 0x55d5e80d9030>,), name='RequirementFlag', qualname='RequirementFlag', mkw=mkw@entry={}, | |
| modname='gssapi.raw.types', doc=0x0) at gssapi/raw/types.c:13392 | |
| #9 0x00007f221ccb58d5 in __pyx_pymod_exec_types (__pyx_pyinit_module=<optimized out>) at gssapi/raw/types.c:10426 | |
| (gdb) py-bt | |
| Traceback (most recent call first): | |
| <built-in method exec_dynamic of module object at remote 0x7f222130a390> | |
| File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed | |
| File "<frozen importlib._bootstrap_external>", line 1297, in exec_module | |
| File "<frozen importlib._bootstrap>", line 935, in _load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1360, in _find_and_load | |
| <built-in method exec_dynamic of module object at remote 0x7f222130a390> | |
| File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed | |
| File "<frozen importlib._bootstrap_external>", line 1297, in exec_module | |
| File "<frozen importlib._bootstrap>", line 935, in _load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1360, in _find_and_load | |
| <built-in method __import__ of module object at remote 0x7f22212f26f0> | |
| <built-in method exec_dynamic of module object at remote 0x7f222130a390> | |
| File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed | |
| File "<frozen importlib._bootstrap_external>", line 1297, in exec_module | |
| File "<frozen importlib._bootstrap>", line 935, in _load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1360, in _find_and_load | |
| <built-in method exec_dynamic of module object at remote 0x7f222130a390> | |
| File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed | |
| File "<frozen importlib._bootstrap_external>", line 1297, in exec_module | |
| File "<frozen importlib._bootstrap>", line 935, in _load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1360, in _find_and_load | |
| <built-in method __import__ of module object at remote 0x7f22212f26f0> | |
| <built-in method exec_dynamic of module object at remote 0x7f222130a390> | |
| File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed | |
| File "<frozen importlib._bootstrap_external>", line 1297, in exec_module | |
| File "<frozen importlib._bootstrap>", line 935, in _load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked | |
| File "<frozen importlib._bootstrap>", line 1360, in _find_and_load | |
| File "/usr/lib64/python3.12/site-packages/gssapi/raw/__init__.py", line 50, in <module> | |
| from gssapi.raw.creds import * # noqa | |
| (...) | |
| (gdb) frame 6 | |
| #6 0x00007fb1531fa0d2 in PyTuple_GET_SIZE (op={}) at /usr/src/debug/python3.12-3.12.2-2.fc39.x86_64/Include/cpython/tupleobject.h:23 | |
| 23 PyTupleObject *tuple = _PyTuple_CAST(op); | |
| (gdb) frame 7 | |
| #7 0x00007fb1531fad1d in method_vectorcall (method=<method at remote 0x7fb14e8b44d0>, args=0x7fb14fdd85a8, nargsf=2, kwnames={}) | |
| at /usr/src/debug/python3.12-3.12.2-2.fc39.x86_64/Objects/classobject.c:66 | |
| (gdb) list | |
| 61 result = _PyObject_VectorcallTstate(tstate, func, newargs, | |
| 62 nargs, kwnames); | |
| 63 newargs[0] = tmp; | |
| 64 } | |
| 65 else { | |
| 66 Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); <== HERE | |
| 67 Py_ssize_t totalargs = nargs + nkwargs; | |
| 68 if (totalargs == 0) { | |
| 69 return _PyObject_VectorcallTstate(tstate, func, &self, 1, NULL); | |
| 70 } | |
| (gdb) frame 8 | |
| #8 0x00007fb14e63b4b7 in __Pyx_Py3MetaclassPrepare (metaclass=metaclass@entry=<type at remote 0x7fb1480df6f0>, | |
| bases=bases@entry=(<EnumType(_generate_next_value_=<staticmethod at remote 0x7fb152b2a1c0>, __module__='enum', __doc__='\n Enum where members are also (and must be) ints\n ', _new_member_=<built-in method __new__ of type object at remote 0x7fb153750d00>, _use_args_=True, _member_names_=[], _member_map_={}, _value2member_map_={}, _unhashable_values_=[], _member_type_=<type at remote 0x7fb153750d00>, _value_repr_=<wrapper_descriptor at remote 0x7fb152c22d50>, __dict__=<getset_descriptor at remote 0x7fb1529d4890>, __format__=<method_descriptor at remote 0x7fb152c241d0>, __str__=<wrapper_descriptor at remote 0x7fb152c22d50>, __repr__=<function at remote 0x7fb1529cec90>, __new__=<function at remote 0x7fb1529ce990>) at remote 0x5563182d0030>,), name='RequirementFlag', qualname='RequirementFlag', mkw=mkw@entry={}, | |
| modname='gssapi.raw.types', doc=0x0) at gssapi/raw/types.c:13392 | |
| (gdb) list | |
| 13387 PyObject *ns; | |
| 13388 if (metaclass) { | |
| 13389 PyObject *prep = __Pyx_PyObject_GetAttrStrNoError(metaclass, __pyx_n_s_prepare); | |
| 13390 if (prep) { | |
| 13391 PyObject *pargs[3] = {NULL, name, bases}; | |
| 13392 ns = __Pyx_PyObject_FastCallDict(prep, pargs+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, mkw); <==== HERE === | |
| 13393 Py_DECREF(prep); | |
| 13394 } else { | |
| 13395 if (unlikely(PyErr_Occurred())) | |
| 13396 return NULL; | |
| ``` | |
| The bug occurs on `from gssapi.raw.creds import * # noqa`: when the C extension `gssapi` is imported, Cython creates types in the module exec function (`__pyx_pymod_exec_types()`). | |
| Problem: `__Pyx_Py3MetaclassPrepare()` pass a dictionary as *kwnames* to `method_vectorcall()`, whereas *kwnames* must be a tuple. Later, an assertion fails since we get the wrong type. | |
| __Pyx_Py3MetaclassPrepare() pass its `mkw` argument to `method_vectorcall()` as `kwnames`: | |
| * frame 9, __pyx_pymod_exec_types(): *__pyx_t_4* | |
| * frame 8, __Pyx_Py3MetaclassPrepare(): *mkw* | |
| * frame 7, method_vectorcall(): *kwnames* | |
| __pyx_pymod_exec_types() creates it as a dictionary with: | |
| ```c | |
| __pyx_t_4 = __Pyx_PyDict_NewPresized(0); | |
| ``` |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment