From b349bd7531e7fbd0d7ab0bece576fd3fefad75a1 Mon Sep 17 00:00:00 2001 From: Anuj Bharambe Date: Mon, 4 May 2026 10:26:50 +0530 Subject: [PATCH 1/8] gh-149180: Avoid double checking tp_as_number, tp_as_sequence, tp_as_mapping During PyType_Ready, assign NULL tp_as_number, tp_as_sequence, and tp_as_mapping pointers to shared static const empty (all-zero) structs. After initialization, these three fields are guaranteed non-NULL for all ready types. Remove redundant NULL checks at all callsites across abstract.c, object.c, bytesobject.c, complexobject.c, floatobject.c, typeobject.c, _bisectmodule.c, and pycore_abstract.h. --- Include/internal/pycore_abstract.h | 2 +- Modules/_bisectmodule.c | 4 +- Objects/abstract.c | 194 +++++++++++++---------------- Objects/bytesobject.c | 2 +- Objects/complexobject.c | 13 +- Objects/floatobject.c | 4 +- Objects/object.c | 9 +- Objects/typeobject.c | 38 ++++-- 8 files changed, 127 insertions(+), 139 deletions(-) diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index b9eb4fd9891e66..7006266815699d 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -13,7 +13,7 @@ static inline int _PyIndex_Check(PyObject *obj) { PyNumberMethods *tp_as_number = Py_TYPE(obj)->tp_as_number; - return (tp_as_number != NULL && tp_as_number->nb_index != NULL); + return (tp_as_number->nb_index != NULL); } // Exported for external JIT support diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 329aa8e117ec3c..9057323e5e0e5a 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -35,11 +35,11 @@ get_sq_item(PyObject *s) // The parts of PySequence_GetItem that we only need to do once PyTypeObject *tp = Py_TYPE(s); PySequenceMethods *m = tp->tp_as_sequence; - if (m && m->sq_item) { + if (m->sq_item) { return m->sq_item; } const char *msg; - if (tp->tp_as_mapping && tp->tp_as_mapping->mp_subscript) { + if (tp->tp_as_mapping->mp_subscript) { msg = "%.200s is not a sequence"; } else { diff --git a/Objects/abstract.c b/Objects/abstract.c index 0bbf60840a3346..359c6ec26559a6 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -62,7 +62,7 @@ PyObject_Size(PyObject *o) } PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (m && m->sq_length) { + if (m->sq_length) { Py_ssize_t len = m->sq_length(o); assert(_Py_CheckSlotResult(o, "__len__", len >= 0)); return len; @@ -81,8 +81,8 @@ PyObject_Length(PyObject *o) int _PyObject_HasLen(PyObject *o) { - return (Py_TYPE(o)->tp_as_sequence && Py_TYPE(o)->tp_as_sequence->sq_length) || - (Py_TYPE(o)->tp_as_mapping && Py_TYPE(o)->tp_as_mapping->mp_length); + return Py_TYPE(o)->tp_as_sequence->sq_length || + Py_TYPE(o)->tp_as_mapping->mp_length; } /* The length hint function returns a non-negative value from o.__len__() @@ -159,14 +159,14 @@ PyObject_GetItem(PyObject *o, PyObject *key) } PyMappingMethods *m = Py_TYPE(o)->tp_as_mapping; - if (m && m->mp_subscript) { + if (m->mp_subscript) { PyObject *item = m->mp_subscript(o, key); assert(_Py_CheckSlotResult(o, "__getitem__", item != NULL)); return item; } PySequenceMethods *ms = Py_TYPE(o)->tp_as_sequence; - if (ms && ms->sq_item) { + if (ms->sq_item) { if (_PyIndex_Check(key)) { Py_ssize_t key_value; key_value = PyNumber_AsSsize_t(key, PyExc_IndexError); @@ -241,13 +241,13 @@ PyObject_SetItem(PyObject *o, PyObject *key, PyObject *value) } PyMappingMethods *m = Py_TYPE(o)->tp_as_mapping; - if (m && m->mp_ass_subscript) { + if (m->mp_ass_subscript) { int res = m->mp_ass_subscript(o, key, value); assert(_Py_CheckSlotResult(o, "__setitem__", res >= 0)); return res; } - if (Py_TYPE(o)->tp_as_sequence) { + { if (_PyIndex_Check(key)) { Py_ssize_t key_value; key_value = PyNumber_AsSsize_t(key, PyExc_IndexError); @@ -275,13 +275,13 @@ PyObject_DelItem(PyObject *o, PyObject *key) } PyMappingMethods *m = Py_TYPE(o)->tp_as_mapping; - if (m && m->mp_ass_subscript) { + if (m->mp_ass_subscript) { int res = m->mp_ass_subscript(o, key, (PyObject*)NULL); assert(_Py_CheckSlotResult(o, "__delitem__", res >= 0)); return res; } - if (Py_TYPE(o)->tp_as_sequence) { + { if (_PyIndex_Check(key)) { Py_ssize_t key_value; key_value = PyNumber_AsSsize_t(key, PyExc_IndexError); @@ -915,7 +915,7 @@ PyNumber_Check(PyObject *o) if (o == NULL) return 0; PyNumberMethods *nb = Py_TYPE(o)->tp_as_number; - return nb && (nb->nb_index || nb->nb_int || nb->nb_float || PyComplex_Check(o)); + return (nb->nb_index || nb->nb_int || nb->nb_float || PyComplex_Check(o)); } /* Binary operators */ @@ -943,16 +943,10 @@ binary_op1(PyObject *v, PyObject *w, const int op_slot #endif ) { - binaryfunc slotv; - if (Py_TYPE(v)->tp_as_number != NULL) { - slotv = NB_BINOP(Py_TYPE(v)->tp_as_number, op_slot); - } - else { - slotv = NULL; - } + binaryfunc slotv = NB_BINOP(Py_TYPE(v)->tp_as_number, op_slot); binaryfunc slotw; - if (!Py_IS_TYPE(w, Py_TYPE(v)) && Py_TYPE(w)->tp_as_number != NULL) { + if (!Py_IS_TYPE(w, Py_TYPE(v))) { slotw = NB_BINOP(Py_TYPE(w)->tp_as_number, op_slot); if (slotw == slotv) { slotw = NULL; @@ -1037,16 +1031,10 @@ ternary_op(PyObject *v, PyNumberMethods *mv = Py_TYPE(v)->tp_as_number; PyNumberMethods *mw = Py_TYPE(w)->tp_as_number; - ternaryfunc slotv; - if (mv != NULL) { - slotv = NB_TERNOP(mv, op_slot); - } - else { - slotv = NULL; - } + ternaryfunc slotv = NB_TERNOP(mv, op_slot); ternaryfunc slotw; - if (!Py_IS_TYPE(w, Py_TYPE(v)) && mw != NULL) { + if (!Py_IS_TYPE(w, Py_TYPE(v))) { slotw = NB_TERNOP(mw, op_slot); if (slotw == slotv) { slotw = NULL; @@ -1083,19 +1071,17 @@ ternary_op(PyObject *v, } PyNumberMethods *mz = Py_TYPE(z)->tp_as_number; - if (mz != NULL) { - ternaryfunc slotz = NB_TERNOP(mz, op_slot); - if (slotz == slotv || slotz == slotw) { - slotz = NULL; - } - if (slotz) { - PyObject *x = slotz(v, w, z); - assert(_Py_CheckSlotResult(z, op_name, x != NULL)); - if (x != Py_NotImplemented) { - return x; - } - Py_DECREF(x); /* can't do it */ + ternaryfunc slotz = NB_TERNOP(mz, op_slot); + if (slotz == slotv || slotz == slotw) { + slotz = NULL; + } + if (slotz) { + PyObject *x = slotz(v, w, z); + assert(_Py_CheckSlotResult(z, op_name, x != NULL)); + if (x != Py_NotImplemented) { + return x; } + Py_DECREF(x); /* can't do it */ } if (z == Py_None) { @@ -1144,7 +1130,7 @@ PyNumber_Add(PyObject *v, PyObject *w) Py_DECREF(result); PySequenceMethods *m = Py_TYPE(v)->tp_as_sequence; - if (m && m->sq_concat) { + if (m->sq_concat) { result = (*m->sq_concat)(v, w); assert(_Py_CheckSlotResult(v, "+", result != NULL)); return result; @@ -1180,10 +1166,10 @@ PyNumber_Multiply(PyObject *v, PyObject *w) PySequenceMethods *mv = Py_TYPE(v)->tp_as_sequence; PySequenceMethods *mw = Py_TYPE(w)->tp_as_sequence; Py_DECREF(result); - if (mv && mv->sq_repeat) { + if (mv->sq_repeat) { return sequence_repeat(mv->sq_repeat, v, w); } - else if (mw && mw->sq_repeat) { + else if (mw->sq_repeat) { return sequence_repeat(mw->sq_repeat, w, v); } result = binop_type_error(v, w, "*"); @@ -1232,16 +1218,14 @@ binary_iop1(PyObject *v, PyObject *w, const int iop_slot, const int op_slot ) { PyNumberMethods *mv = Py_TYPE(v)->tp_as_number; - if (mv != NULL) { - binaryfunc slot = NB_BINOP(mv, iop_slot); - if (slot) { - PyObject *x = (slot)(v, w); - assert(_Py_CheckSlotResult(v, op_name, x != NULL)); - if (x != Py_NotImplemented) { - return x; - } - Py_DECREF(x); + binaryfunc slot = NB_BINOP(mv, iop_slot); + if (slot) { + PyObject *x = (slot)(v, w); + assert(_Py_CheckSlotResult(v, op_name, x != NULL)); + if (x != Py_NotImplemented) { + return x; } + Py_DECREF(x); } #ifdef NDEBUG return binary_op1(v, w, op_slot); @@ -1273,15 +1257,13 @@ ternary_iop(PyObject *v, PyObject *w, PyObject *z, const int iop_slot, const int const char *op_name) { PyNumberMethods *mv = Py_TYPE(v)->tp_as_number; - if (mv != NULL) { - ternaryfunc slot = NB_TERNOP(mv, iop_slot); - if (slot) { - PyObject *x = (slot)(v, w, z); - if (x != Py_NotImplemented) { - return x; - } - Py_DECREF(x); + ternaryfunc slot = NB_TERNOP(mv, iop_slot); + if (slot) { + PyObject *x = (slot)(v, w, z); + if (x != Py_NotImplemented) { + return x; } + Py_DECREF(x); } return ternary_op(v, w, z, op_slot, op_name); } @@ -1311,15 +1293,13 @@ PyNumber_InPlaceAdd(PyObject *v, PyObject *w) if (result == Py_NotImplemented) { PySequenceMethods *m = Py_TYPE(v)->tp_as_sequence; Py_DECREF(result); - if (m != NULL) { - binaryfunc func = m->sq_inplace_concat; - if (func == NULL) - func = m->sq_concat; - if (func != NULL) { - result = func(v, w); - assert(_Py_CheckSlotResult(v, "+=", result != NULL)); - return result; - } + binaryfunc func = m->sq_inplace_concat; + if (func == NULL) + func = m->sq_concat; + if (func != NULL) { + result = func(v, w); + assert(_Py_CheckSlotResult(v, "+=", result != NULL)); + return result; } result = binop_type_error(v, w, "+="); } @@ -1336,20 +1316,16 @@ PyNumber_InPlaceMultiply(PyObject *v, PyObject *w) PySequenceMethods *mv = Py_TYPE(v)->tp_as_sequence; PySequenceMethods *mw = Py_TYPE(w)->tp_as_sequence; Py_DECREF(result); - if (mv != NULL) { - f = mv->sq_inplace_repeat; - if (f == NULL) - f = mv->sq_repeat; - if (f != NULL) - return sequence_repeat(f, v, w); - } - else if (mw != NULL) { - /* Note that the right hand operand should not be - * mutated in this case so sq_inplace_repeat is not - * used. */ - if (mw->sq_repeat) - return sequence_repeat(mw->sq_repeat, w, v); - } + f = mv->sq_inplace_repeat; + if (f == NULL) + f = mv->sq_repeat; + if (f != NULL) + return sequence_repeat(f, v, w); + /* Note that the right hand operand should not be + * mutated in this case so sq_inplace_repeat is not + * used. */ + if (mw->sq_repeat) + return sequence_repeat(mw->sq_repeat, w, v); result = binop_type_error(v, w, "*="); } return result; @@ -1379,7 +1355,7 @@ _PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs) } \ \ PyNumberMethods *m = Py_TYPE(o)->tp_as_number; \ - if (m && m->op) { \ + if (m->op) { \ PyObject *res = (*m->op)(o); \ assert(_Py_CheckSlotResult(o, #meth_name, res != NULL)); \ return res; \ @@ -1530,7 +1506,7 @@ PyNumber_Long(PyObject *o) return Py_NewRef(o); } m = Py_TYPE(o)->tp_as_number; - if (m && m->nb_int) { /* This should include subclasses of int */ + if (m->nb_int) { /* This should include subclasses of int */ /* Convert using the nb_int slot, which should return something of exact type int. */ result = m->nb_int(o); @@ -1558,7 +1534,7 @@ PyNumber_Long(PyObject *o) Py_SETREF(result, _PyLong_Copy((PyLongObject *)result)); return result; } - if (m && m->nb_index) { + if (m->nb_index) { return PyNumber_Index(o); } @@ -1610,7 +1586,7 @@ PyNumber_Float(PyObject *o) } PyNumberMethods *m = Py_TYPE(o)->tp_as_number; - if (m && m->nb_float) { /* This should include subclasses of float */ + if (m->nb_float) { /* This should include subclasses of float */ PyObject *res = m->nb_float(o); assert(_Py_CheckSlotResult(o, "__float__", res != NULL)); if (!res || PyFloat_CheckExact(res)) { @@ -1637,7 +1613,7 @@ PyNumber_Float(PyObject *o) return PyFloat_FromDouble(val); } - if (m && m->nb_index) { + if (m->nb_index) { PyObject *res = _PyNumber_Index(o); if (!res) { return NULL; @@ -1682,8 +1658,7 @@ PySequence_Check(PyObject *s) { if (PyDict_Check(s)) return 0; - return Py_TYPE(s)->tp_as_sequence && - Py_TYPE(s)->tp_as_sequence->sq_item != NULL; + return Py_TYPE(s)->tp_as_sequence->sq_item != NULL; } Py_ssize_t @@ -1695,13 +1670,13 @@ PySequence_Size(PyObject *s) } PySequenceMethods *m = Py_TYPE(s)->tp_as_sequence; - if (m && m->sq_length) { + if (m->sq_length) { Py_ssize_t len = m->sq_length(s); assert(_Py_CheckSlotResult(s, "__len__", len >= 0)); return len; } - if (Py_TYPE(s)->tp_as_mapping && Py_TYPE(s)->tp_as_mapping->mp_length) { + if (Py_TYPE(s)->tp_as_mapping->mp_length) { type_error("%.200s is not a sequence", s); return -1; } @@ -1725,7 +1700,7 @@ PySequence_Concat(PyObject *s, PyObject *o) } PySequenceMethods *m = Py_TYPE(s)->tp_as_sequence; - if (m && m->sq_concat) { + if (m->sq_concat) { PyObject *res = m->sq_concat(s, o); assert(_Py_CheckSlotResult(s, "+", res != NULL)); return res; @@ -1751,7 +1726,7 @@ PySequence_Repeat(PyObject *o, Py_ssize_t count) } PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (m && m->sq_repeat) { + if (m->sq_repeat) { PyObject *res = m->sq_repeat(o, count); assert(_Py_CheckSlotResult(o, "*", res != NULL)); return res; @@ -1782,12 +1757,12 @@ PySequence_InPlaceConcat(PyObject *s, PyObject *o) } PySequenceMethods *m = Py_TYPE(s)->tp_as_sequence; - if (m && m->sq_inplace_concat) { + if (m->sq_inplace_concat) { PyObject *res = m->sq_inplace_concat(s, o); assert(_Py_CheckSlotResult(s, "+=", res != NULL)); return res; } - if (m && m->sq_concat) { + if (m->sq_concat) { PyObject *res = m->sq_concat(s, o); assert(_Py_CheckSlotResult(s, "+", res != NULL)); return res; @@ -1811,12 +1786,12 @@ PySequence_InPlaceRepeat(PyObject *o, Py_ssize_t count) } PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (m && m->sq_inplace_repeat) { + if (m->sq_inplace_repeat) { PyObject *res = m->sq_inplace_repeat(o, count); assert(_Py_CheckSlotResult(o, "*=", res != NULL)); return res; } - if (m && m->sq_repeat) { + if (m->sq_repeat) { PyObject *res = m->sq_repeat(o, count); assert(_Py_CheckSlotResult(o, "*", res != NULL)); return res; @@ -1845,7 +1820,7 @@ PySequence_GetItem(PyObject *s, Py_ssize_t i) } PySequenceMethods *m = Py_TYPE(s)->tp_as_sequence; - if (m && m->sq_item) { + if (m->sq_item) { if (i < 0) { if (m->sq_length) { Py_ssize_t l = (*m->sq_length)(s); @@ -1861,7 +1836,7 @@ PySequence_GetItem(PyObject *s, Py_ssize_t i) return res; } - if (Py_TYPE(s)->tp_as_mapping && Py_TYPE(s)->tp_as_mapping->mp_subscript) { + if (Py_TYPE(s)->tp_as_mapping->mp_subscript) { return type_error("%.200s is not a sequence", s); } return type_error("'%.200s' object does not support indexing", s); @@ -1875,7 +1850,7 @@ PySequence_GetSlice(PyObject *s, Py_ssize_t i1, Py_ssize_t i2) } PyMappingMethods *mp = Py_TYPE(s)->tp_as_mapping; - if (mp && mp->mp_subscript) { + if (mp->mp_subscript) { PyObject *slice = _PySlice_FromIndices(i1, i2); if (!slice) { return NULL; @@ -1898,7 +1873,7 @@ PySequence_SetItem(PyObject *s, Py_ssize_t i, PyObject *o) } PySequenceMethods *m = Py_TYPE(s)->tp_as_sequence; - if (m && m->sq_ass_item) { + if (m->sq_ass_item) { if (i < 0) { if (m->sq_length) { Py_ssize_t l = (*m->sq_length)(s); @@ -1914,7 +1889,7 @@ PySequence_SetItem(PyObject *s, Py_ssize_t i, PyObject *o) return res; } - if (Py_TYPE(s)->tp_as_mapping && Py_TYPE(s)->tp_as_mapping->mp_ass_subscript) { + if (Py_TYPE(s)->tp_as_mapping->mp_ass_subscript) { type_error("%.200s is not a sequence", s); return -1; } @@ -1931,7 +1906,7 @@ PySequence_DelItem(PyObject *s, Py_ssize_t i) } PySequenceMethods *m = Py_TYPE(s)->tp_as_sequence; - if (m && m->sq_ass_item) { + if (m->sq_ass_item) { if (i < 0) { if (m->sq_length) { Py_ssize_t l = (*m->sq_length)(s); @@ -1947,7 +1922,7 @@ PySequence_DelItem(PyObject *s, Py_ssize_t i) return res; } - if (Py_TYPE(s)->tp_as_mapping && Py_TYPE(s)->tp_as_mapping->mp_ass_subscript) { + if (Py_TYPE(s)->tp_as_mapping->mp_ass_subscript) { type_error("%.200s is not a sequence", s); return -1; } @@ -1964,7 +1939,7 @@ PySequence_SetSlice(PyObject *s, Py_ssize_t i1, Py_ssize_t i2, PyObject *o) } PyMappingMethods *mp = Py_TYPE(s)->tp_as_mapping; - if (mp && mp->mp_ass_subscript) { + if (mp->mp_ass_subscript) { PyObject *slice = _PySlice_FromIndices(i1, i2); if (!slice) return -1; @@ -1987,7 +1962,7 @@ PySequence_DelSlice(PyObject *s, Py_ssize_t i1, Py_ssize_t i2) } PyMappingMethods *mp = Py_TYPE(s)->tp_as_mapping; - if (mp && mp->mp_ass_subscript) { + if (mp->mp_ass_subscript) { PyObject *slice = _PySlice_FromIndices(i1, i2); if (!slice) { return -1; @@ -2240,7 +2215,7 @@ int PySequence_Contains(PyObject *seq, PyObject *ob) { PySequenceMethods *sqm = Py_TYPE(seq)->tp_as_sequence; - if (sqm != NULL && sqm->sq_contains != NULL) { + if (sqm->sq_contains != NULL) { int res = (*sqm->sq_contains)(seq, ob); assert(_Py_CheckSlotResult(seq, "__contains__", res >= 0)); return res; @@ -2268,8 +2243,7 @@ PySequence_Index(PyObject *s, PyObject *o) int PyMapping_Check(PyObject *o) { - return o && Py_TYPE(o)->tp_as_mapping && - Py_TYPE(o)->tp_as_mapping->mp_subscript; + return o && Py_TYPE(o)->tp_as_mapping->mp_subscript; } Py_ssize_t @@ -2281,13 +2255,13 @@ PyMapping_Size(PyObject *o) } PyMappingMethods *m = Py_TYPE(o)->tp_as_mapping; - if (m && m->mp_length) { + if (m->mp_length) { Py_ssize_t len = m->mp_length(o); assert(_Py_CheckSlotResult(o, "__len__", len >= 0)); return len; } - if (Py_TYPE(o)->tp_as_sequence && Py_TYPE(o)->tp_as_sequence->sq_length) { + if (Py_TYPE(o)->tp_as_sequence->sq_length) { type_error("%.200s is not a mapping", o); return -1; } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 8a9d1b133affb3..ede5e94367b286 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -661,7 +661,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, arglen = -1; argidx = -2; } - if (Py_TYPE(args)->tp_as_mapping && Py_TYPE(args)->tp_as_mapping->mp_subscript && + if (Py_TYPE(args)->tp_as_mapping->mp_subscript && !PyTuple_Check(args) && !PyBytes_Check(args) && !PyUnicode_Check(args) && !PyByteArray_Check(args)) { dict = args; diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 3612c2699a557d..dffe09169e0b5f 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -1149,8 +1149,8 @@ actual_complex_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_complex c = ((PyComplexObject*)arg)->cval; res = complex_subtype_from_doubles(type, c.real, c.imag); } - else if ((nbr = Py_TYPE(arg)->tp_as_number) != NULL && - (nbr->nb_float != NULL || nbr->nb_index != NULL)) + else if ((nbr = Py_TYPE(arg)->tp_as_number)->nb_float != NULL || + nbr->nb_index != NULL) { /* The argument really is entirely real, and contributes nothing in the imaginary direction. @@ -1210,8 +1210,7 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) } nbr = Py_TYPE(r)->tp_as_number; - if (nbr == NULL || - (nbr->nb_float == NULL && nbr->nb_index == NULL && !PyComplex_Check(r))) + if (nbr->nb_float == NULL && nbr->nb_index == NULL && !PyComplex_Check(r)) { PyErr_Format(PyExc_TypeError, "complex() argument 'real' must be a real number, not %T", @@ -1223,8 +1222,7 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) } if (i != NULL) { nbi = Py_TYPE(i)->tp_as_number; - if (nbi == NULL || - (nbi->nb_float == NULL && nbi->nb_index == NULL && !PyComplex_Check(i))) + if (nbi->nb_float == NULL && nbi->nb_index == NULL && !PyComplex_Check(i)) { PyErr_Format(PyExc_TypeError, "complex() argument 'imag' must be a real number, not %T", @@ -1256,8 +1254,7 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) Py_DECREF(r); } nbr = Py_TYPE(orig_r)->tp_as_number; - if (nbr == NULL || - (nbr->nb_float == NULL && nbr->nb_index == NULL)) + if (nbr->nb_float == NULL && nbr->nb_index == NULL) { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, "complex() argument 'real' must be a real number, not %T", diff --git a/Objects/floatobject.c b/Objects/floatobject.c index d91468dddded9b..f70473296dd7dd 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -260,8 +260,8 @@ PyFloat_AsDouble(PyObject *op) } nb = Py_TYPE(op)->tp_as_number; - if (nb == NULL || nb->nb_float == NULL) { - if (nb && nb->nb_index) { + if (nb->nb_float == NULL) { + if (nb->nb_index) { PyObject *res = _PyNumber_Index(op); if (!res) { return -1; diff --git a/Objects/object.c b/Objects/object.c index e0e26bb50d3653..a9d80df1f27a5d 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2144,14 +2144,11 @@ PyObject_IsTrue(PyObject *v) return 0; if (v == Py_None) return 0; - else if (Py_TYPE(v)->tp_as_number != NULL && - Py_TYPE(v)->tp_as_number->nb_bool != NULL) + else if (Py_TYPE(v)->tp_as_number->nb_bool != NULL) res = (*Py_TYPE(v)->tp_as_number->nb_bool)(v); - else if (Py_TYPE(v)->tp_as_mapping != NULL && - Py_TYPE(v)->tp_as_mapping->mp_length != NULL) + else if (Py_TYPE(v)->tp_as_mapping->mp_length != NULL) res = (*Py_TYPE(v)->tp_as_mapping->mp_length)(v); - else if (Py_TYPE(v)->tp_as_sequence != NULL && - Py_TYPE(v)->tp_as_sequence->sq_length != NULL) + else if (Py_TYPE(v)->tp_as_sequence->sq_length != NULL) res = (*Py_TYPE(v)->tp_as_sequence->sq_length)(v); else return 1; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index fb3c7101410683..3c69a60a930503 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -896,6 +896,10 @@ _PyType_CheckConsistency(PyTypeObject *type) CHECK(PyDict_Contains(lookup_tp_dict(type), &_Py_ID(__new__)) == 0); } + CHECK(type->tp_as_number != NULL); + CHECK(type->tp_as_sequence != NULL); + CHECK(type->tp_as_mapping != NULL); + return 1; #undef CHECK } @@ -8718,7 +8722,8 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) if (type->tp_as_number != NULL && base->tp_as_number != NULL) { basebase = base->tp_base; - if (basebase->tp_as_number == NULL) + // basebase is NULL when base is 'object' (tp_base == NULL). + if (basebase == NULL || basebase->tp_as_number == NULL) basebase = NULL; COPYNUM(nb_add); COPYNUM(nb_subtract); @@ -8768,7 +8773,8 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) if (type->tp_as_sequence != NULL && base->tp_as_sequence != NULL) { basebase = base->tp_base; - if (basebase->tp_as_sequence == NULL) + // basebase is NULL when base is 'object' (tp_base == NULL). + if (basebase == NULL || basebase->tp_as_sequence == NULL) basebase = NULL; COPYSEQ(sq_length); COPYSEQ(sq_concat); @@ -8782,7 +8788,8 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) if (type->tp_as_mapping != NULL && base->tp_as_mapping != NULL) { basebase = base->tp_base; - if (basebase->tp_as_mapping == NULL) + // basebase is NULL when base is 'object' (tp_base == NULL). + if (basebase == NULL || basebase->tp_as_mapping == NULL) basebase = NULL; COPYMAP(mp_length); COPYMAP(mp_subscript); @@ -9155,6 +9162,10 @@ type_ready_mro(PyTypeObject *type, int initial) } +static const PyNumberMethods _Py_empty_number_methods = {0}; +static const PySequenceMethods _Py_empty_sequence_methods = {0}; +static const PyMappingMethods _Py_empty_mapping_methods = {0}; + // For static types, inherit tp_as_xxx structures from the base class // if it's NULL. // @@ -9216,6 +9227,19 @@ type_ready_inherit(PyTypeObject *type) type_ready_inherit_as_structs(type, base); } + // Ensure tp_as_number, tp_as_sequence, and tp_as_mapping are never + // NULL after PyType_Ready. Types that don't provide their own struct + // share a common empty one. + if (type->tp_as_number == NULL) { + type->tp_as_number = (PyNumberMethods *)&_Py_empty_number_methods; + } + if (type->tp_as_sequence == NULL) { + type->tp_as_sequence = (PySequenceMethods *)&_Py_empty_sequence_methods; + } + if (type->tp_as_mapping == NULL) { + type->tp_as_mapping = (PyMappingMethods *)&_Py_empty_mapping_methods; + } + /* Sanity check for tp_free. */ if (_PyType_IS_GC(type) && (type->tp_flags & Py_TPFLAGS_BASETYPE) && (type->tp_free == NULL || type->tp_free == PyObject_Free)) @@ -10447,10 +10471,8 @@ FUNCNAME(PyObject *self, PyObject *other) \ PyObject* stack[2]; \ PyThreadState *tstate = _PyThreadState_GET(); \ int do_other = !Py_IS_TYPE(self, Py_TYPE(other)) && \ - Py_TYPE(other)->tp_as_number != NULL && \ Py_TYPE(other)->tp_as_number->SLOTNAME == TESTFUNC; \ - if (Py_TYPE(self)->tp_as_number != NULL && \ - Py_TYPE(self)->tp_as_number->SLOTNAME == TESTFUNC) { \ + if (Py_TYPE(self)->tp_as_number->SLOTNAME == TESTFUNC) { \ PyObject *r; \ if (do_other && PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self))) { \ int ok = method_is_overloaded(self, other, &_Py_ID(RDUNDER)); \ @@ -10627,10 +10649,8 @@ slot_nb_power(PyObject *self, PyObject *other, PyObject *modulus) PyObject* stack[3]; PyThreadState *tstate = _PyThreadState_GET(); int do_other = !Py_IS_TYPE(self, Py_TYPE(other)) && - Py_TYPE(other)->tp_as_number != NULL && Py_TYPE(other)->tp_as_number->nb_power == slot_nb_power; - if (Py_TYPE(self)->tp_as_number != NULL && - Py_TYPE(self)->tp_as_number->nb_power == slot_nb_power) { + if (Py_TYPE(self)->tp_as_number->nb_power == slot_nb_power) { PyObject *r; if (do_other && PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self))) { int ok = method_is_overloaded(self, other, &_Py_ID(__rpow__)); From b271c546793e5961ce4e7d38417020898cf644d5 Mon Sep 17 00:00:00 2001 From: Anuj Bharambe Date: Mon, 4 May 2026 10:36:54 +0530 Subject: [PATCH 2/8] Add NEWS.d entry for gh-149180 --- .../2026-05-04-10-35-49.gh-issue-149180.wCERVE.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-04-10-35-49.gh-issue-149180.wCERVE.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-04-10-35-49.gh-issue-149180.wCERVE.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-04-10-35-49.gh-issue-149180.wCERVE.rst new file mode 100644 index 00000000000000..947b844b3f9f7d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-04-10-35-49.gh-issue-149180.wCERVE.rst @@ -0,0 +1,5 @@ +Ensure :c:member:`~PyTypeObject.tp_as_number`, +:c:member:`~PyTypeObject.tp_as_sequence`, and +:c:member:`~PyTypeObject.tp_as_mapping` are never ``NULL`` after +:c:func:`PyType_Ready` by assigning shared empty structs as fallbacks. +Remove redundant ``NULL`` checks at callsites. From 6c11104048c7c511abc578d0cda329e8a34c3ffe Mon Sep 17 00:00:00 2001 From: Anuj Bharambe Date: Mon, 4 May 2026 11:03:48 +0530 Subject: [PATCH 3/8] Add _Py_empty_*_methods to globals-to-fix.tsv --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index db575d870be5c5..128ee3b05b4f74 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -93,6 +93,9 @@ Objects/templateobject.c - _PyTemplateIter_Type - Objects/templateobject.c - _PyTemplate_Type - Objects/tupleobject.c - PyTupleIter_Type - Objects/tupleobject.c - PyTuple_Type - +Objects/typeobject.c - _Py_empty_mapping_methods - +Objects/typeobject.c - _Py_empty_number_methods - +Objects/typeobject.c - _Py_empty_sequence_methods - Objects/typeobject.c - _PyBufferWrapper_Type - Objects/typeobject.c - PyBaseObject_Type - Objects/typeobject.c - PySuper_Type - From 6efcd85df49ff5b8dc1ca2437f3e1c984ee82bf8 Mon Sep 17 00:00:00 2001 From: Anuj Bharambe Date: Mon, 4 May 2026 11:53:40 +0530 Subject: [PATCH 4/8] Move _Py_empty_*_methods to ignored.tsv (they are const) --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 5 +---- Tools/c-analyzer/cpython/ignored.tsv | 3 +++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 128ee3b05b4f74..d684c01a5abe7b 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -92,10 +92,7 @@ Objects/sliceobject.c - PySlice_Type - Objects/templateobject.c - _PyTemplateIter_Type - Objects/templateobject.c - _PyTemplate_Type - Objects/tupleobject.c - PyTupleIter_Type - -Objects/tupleobject.c - PyTuple_Type - -Objects/typeobject.c - _Py_empty_mapping_methods - -Objects/typeobject.c - _Py_empty_number_methods - -Objects/typeobject.c - _Py_empty_sequence_methods - +Objects/tupleobject.c - PyTuple_Type - Objects/typeobject.c - _PyBufferWrapper_Type - Objects/typeobject.c - PyBaseObject_Type - Objects/typeobject.c - PySuper_Type - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index aa89e312b62482..c3cb1693c88b91 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -352,6 +352,9 @@ Objects/typeobject.c - name_op - Objects/typeobject.c - slotdefs - # It initialized only once when main interpeter starts Objects/typeobject.c - slotdefs_dups - +Objects/typeobject.c - _Py_empty_mapping_methods - +Objects/typeobject.c - _Py_empty_number_methods - +Objects/typeobject.c - _Py_empty_sequence_methods - Objects/unicodeobject.c - stripfuncnames - Objects/unicodeobject.c - utf7_category - Objects/unicodeobject.c unicode_decode_call_errorhandler_wchar argparse - From 54ec99580b6f283be47dbf0bb1fde96af75b2c49 Mon Sep 17 00:00:00 2001 From: Anuj Bharambe Date: Mon, 4 May 2026 12:27:31 +0530 Subject: [PATCH 5/8] Fix tabs in ignored.tsv, revert whitespace in globals-to-fix.tsv --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 2 +- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index d684c01a5abe7b..db575d870be5c5 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -92,7 +92,7 @@ Objects/sliceobject.c - PySlice_Type - Objects/templateobject.c - _PyTemplateIter_Type - Objects/templateobject.c - _PyTemplate_Type - Objects/tupleobject.c - PyTupleIter_Type - -Objects/tupleobject.c - PyTuple_Type - +Objects/tupleobject.c - PyTuple_Type - Objects/typeobject.c - _PyBufferWrapper_Type - Objects/typeobject.c - PyBaseObject_Type - Objects/typeobject.c - PySuper_Type - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index c3cb1693c88b91..98d303e6e56dd6 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -354,7 +354,7 @@ Objects/typeobject.c - slotdefs - Objects/typeobject.c - slotdefs_dups - Objects/typeobject.c - _Py_empty_mapping_methods - Objects/typeobject.c - _Py_empty_number_methods - -Objects/typeobject.c - _Py_empty_sequence_methods - +Objects/typeobject.c - _Py_empty_sequence_methods - Objects/unicodeobject.c - stripfuncnames - Objects/unicodeobject.c - utf7_category - Objects/unicodeobject.c unicode_decode_call_errorhandler_wchar argparse - From 42341bce4abf329df088f16b4322187c8b526dd4 Mon Sep 17 00:00:00 2001 From: Anuj Bharambe Date: Mon, 4 May 2026 13:07:17 +0530 Subject: [PATCH 6/8] Fix tabs in ignored.tsv --- Tools/c-analyzer/cpython/ignored.tsv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 98d303e6e56dd6..8c5874472616a4 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -352,9 +352,9 @@ Objects/typeobject.c - name_op - Objects/typeobject.c - slotdefs - # It initialized only once when main interpeter starts Objects/typeobject.c - slotdefs_dups - -Objects/typeobject.c - _Py_empty_mapping_methods - -Objects/typeobject.c - _Py_empty_number_methods - -Objects/typeobject.c - _Py_empty_sequence_methods - +Objects/typeobject.c - _Py_empty_mapping_methods - +Objects/typeobject.c - _Py_empty_number_methods - +Objects/typeobject.c - _Py_empty_sequence_methods - Objects/unicodeobject.c - stripfuncnames - Objects/unicodeobject.c - utf7_category - Objects/unicodeobject.c unicode_decode_call_errorhandler_wchar argparse - From 739fdfd9ed7bc996d9c4789737d6846740badb77 Mon Sep 17 00:00:00 2001 From: Anuj Bharambe Date: Mon, 4 May 2026 13:20:33 +0530 Subject: [PATCH 7/8] Fix remaining tabs in ignored.tsv --- Tools/c-analyzer/cpython/ignored.tsv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 8c5874472616a4..1b2cb84f9a81c4 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -352,9 +352,9 @@ Objects/typeobject.c - name_op - Objects/typeobject.c - slotdefs - # It initialized only once when main interpeter starts Objects/typeobject.c - slotdefs_dups - -Objects/typeobject.c - _Py_empty_mapping_methods - -Objects/typeobject.c - _Py_empty_number_methods - -Objects/typeobject.c - _Py_empty_sequence_methods - +Objects/typeobject.c - _Py_empty_mapping_methods - +Objects/typeobject.c - _Py_empty_number_methods - +Objects/typeobject.c - _Py_empty_sequence_methods - Objects/unicodeobject.c - stripfuncnames - Objects/unicodeobject.c - utf7_category - Objects/unicodeobject.c unicode_decode_call_errorhandler_wchar argparse - From 8f07417e64a18c908ce34a0e0df6b9c20b5e287e Mon Sep 17 00:00:00 2001 From: Anuj Bharambe Date: Wed, 6 May 2026 12:45:56 +0530 Subject: [PATCH 8/8] Address reviewer feedback on tp_as_* empty struct fallbacks * Give empty structs external linkage (const, not static const) and add extern declarations in pycore_object.h so other translation units can perform semantic "has real methods" checks. * Fix inherit_slots guards: use `!= &_Py_empty_*` instead of the `basebase == NULL` workaround, since object now carries the empty structs. * Restore if/else if semantics in PyNumber_InPlaceMultiply to avoid checking both operands when the left one has sequence methods. * Replace bare braces in PyObject_SetItem/DelItem with proper `!= &_Py_empty_sequence_methods` semantic checks. * Add Category A simplifications in Python/specialize.c (check_type_always_true). * Remove c-analyzer ignored.tsv entries (no longer needed with external linkage). --- .github/workflows/build.yml | 16 +- .github/workflows/reusable-ubuntu.yml | 2 +- Doc/deprecations/pending-removal-in-3.18.rst | 6 - Doc/deprecations/pending-removal-in-3.20.rst | 10 - Doc/library/pkgutil.rst | 12 +- Doc/library/profiling.sampling.rst | 122 +--- Doc/library/site.rst | 184 +----- Doc/library/typing.rst | 2 +- Doc/whatsnew/3.15.rst | 4 - Include/internal/pycore_object.h | 4 + Lib/_colorize.py | 21 - Lib/dataclasses.py | 70 +- Lib/pkgutil.py | 44 +- Lib/profiling/sampling/cli.py | 142 +--- Lib/profiling/sampling/collector.py | 164 ++--- Lib/profiling/sampling/sample.py | 55 +- Lib/site.py | 411 ++++-------- Lib/test/test_dataclasses/__init__.py | 23 +- Lib/test/test_httpservers.py | 53 +- Lib/test/test_pkgutil.py | 52 +- .../test_sampling_profiler/test_cli.py | 155 +---- .../test_sampling_profiler/test_profiler.py | 91 +-- Lib/test/test_site.py | 624 ++---------------- Misc/externals.spdx.json | 8 +- Modules/_ssl_data_36.h | 9 +- Objects/abstract.c | 28 +- Objects/typeobject.c | 27 +- PCbuild/get_externals.bat | 4 +- PCbuild/python.props | 4 +- Python/specialize.c | 6 +- Tools/c-analyzer/cpython/ignored.tsv | 3 - Tools/ssl/multissltests.py | 10 +- 32 files changed, 443 insertions(+), 1923 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4397fc7de54a4..33a5950c1483a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -278,13 +278,13 @@ jobs: # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs # supported by important vendors such as AWS-LC. - { name: openssl, version: 1.1.1w } - - { name: openssl, version: 3.0.20 } - - { name: openssl, version: 3.3.7 } - - { name: openssl, version: 3.4.5 } - - { name: openssl, version: 3.5.6 } - - { name: openssl, version: 3.6.2 } + - { name: openssl, version: 3.0.19 } + - { name: openssl, version: 3.3.6 } + - { name: openssl, version: 3.4.4 } + - { name: openssl, version: 3.5.5 } + - { name: openssl, version: 3.6.1 } ## AWS-LC - - { name: aws-lc, version: 1.72.1 } + - { name: aws-lc, version: 1.68.0 } env: SSLLIB_VER: ${{ matrix.ssllib.version }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -398,7 +398,7 @@ jobs: needs: build-context if: needs.build-context.outputs.run-ubuntu == 'true' env: - OPENSSL_VER: 3.5.6 + OPENSSL_VER: 3.5.5 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -506,7 +506,7 @@ jobs: matrix: os: [ubuntu-24.04] env: - OPENSSL_VER: 3.5.6 + OPENSSL_VER: 3.5.5 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index a7e307848af670..87fba6221fb917 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -35,7 +35,7 @@ jobs: runs-on: ${{ inputs.os }} timeout-minutes: 60 env: - OPENSSL_VER: 3.5.6 + OPENSSL_VER: 3.5.5 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: diff --git a/Doc/deprecations/pending-removal-in-3.18.rst b/Doc/deprecations/pending-removal-in-3.18.rst index 19113aab981bbc..eb42fe9919eaeb 100644 --- a/Doc/deprecations/pending-removal-in-3.18.rst +++ b/Doc/deprecations/pending-removal-in-3.18.rst @@ -10,9 +10,3 @@ Pending removal in Python 3.18 specifier ``'N'``, which is only supported in the :mod:`!decimal` module's C implementation, has been deprecated since Python 3.13. (Contributed by Serhiy Storchaka in :gh:`89902`.) - -* Deprecations defined by :pep:`829`: - - * ``import`` lines in :file:`{name}.pth` files are silently ignored. - - (Contributed by Barry Warsaw in :gh:`148641`.) diff --git a/Doc/deprecations/pending-removal-in-3.20.rst b/Doc/deprecations/pending-removal-in-3.20.rst index 011565dfbb090d..12d7acf5ce05b4 100644 --- a/Doc/deprecations/pending-removal-in-3.20.rst +++ b/Doc/deprecations/pending-removal-in-3.20.rst @@ -39,16 +39,6 @@ Pending removal in Python 3.20 (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) -* Deprecations defined by :pep:`829`: - - * Warnings are produced for ``import`` lines found in :file:`{name}.pth` - files. - - * :file:`{name}.pth` files are no longer decoded in the locale encoding by - default. They **MUST** be encoded in ``utf-8-sig``. - - (Contributed by Barry Warsaw in :gh:`148641`.) - * :mod:`ast`: * Creating instances of abstract AST nodes (such as :class:`ast.AST` diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 5473a367c49a3a..aa7dd71c1329df 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -194,7 +194,7 @@ support. The :mod:`importlib.resources` module provides structured access to module resources. -.. function:: resolve_name(name, *, strict=False) +.. function:: resolve_name(name) Resolve a name to an object. @@ -208,7 +208,6 @@ support. * ``W(.W)*`` * ``W(.W)*:(W(.W)*)?`` - * ``W(.W)*:(W(.W)*)`` The first form is intended for backward compatibility only. It assumes that some part of the dotted name is a package, and the rest is an object @@ -223,11 +222,6 @@ support. hierarchy within that package. Only one import is needed in this form. If it ends with the colon, then a module object is returned. - The first two forms are accepted when ``strict=False`` (the default). - - The third form requires both the module name and callable, separated by - a colon. Only this form is accepted when ``strict=True``. - The function will return an object (which might be a module), or raise one of the following exceptions: @@ -239,7 +233,3 @@ support. hierarchy within the imported package to get to the desired object. .. versionadded:: 3.9 - - .. versionchanged:: 3.15 - - The optional keyword-only ``strict`` flag was added. diff --git a/Doc/library/profiling.sampling.rst b/Doc/library/profiling.sampling.rst index aeb1a429b58515..790d36001800d0 100644 --- a/Doc/library/profiling.sampling.rst +++ b/Doc/library/profiling.sampling.rst @@ -153,10 +153,6 @@ Attach to a running process by PID:: python -m profiling.sampling attach 12345 -Print a single snapshot of a running process's stack:: - - python -m profiling.sampling dump 12345 - Use live mode for real-time monitoring (press ``q`` to quit):: python -m profiling.sampling run --live script.py @@ -177,9 +173,8 @@ Enable opcode-level profiling to see which bytecode instructions are executing:: Commands ======== -Tachyon operates through several subcommands. ``run`` and ``attach`` collect -samples over time; ``dump`` captures a single snapshot; ``replay`` converts -binary profiles to other formats. +Tachyon operates through two subcommands that determine how to obtain the +target process. The ``run`` command @@ -222,78 +217,6 @@ On most systems, attaching to another process requires appropriate permissions. See :ref:`profiling-permissions` for platform-specific requirements. -.. _dump-command: - -The ``dump`` command --------------------- - -The ``dump`` command prints a single snapshot of a running process's Python -stack and exits, similar to a traceback:: - - python -m profiling.sampling dump 12345 - -Unlike ``attach``, ``dump`` does not run a sampling loop: it reads the -stack once. This is useful for investigating hung or unresponsive -processes, or for answering "what is this process doing right now?". - -The output mirrors a traceback (most recent call last) and annotates each -thread with its current state (main thread, has GIL, on CPU, waiting for -GIL, has exception, or idle): - -.. code-block:: text - - Stack dump for PID 12345, thread 140735 (main thread, has GIL, on CPU; most recent call last): - File "server.py", line 28, in serve - await handle_request(req) - File "handler.py", line 91, in handle_request - result = expensive_call(req) - -When the target's source files are readable, ``dump`` prints the source -line for each frame and highlights the executing expression. - -Like ``attach``, ``dump`` requires permission to read the target process's -memory. See :ref:`profiling-permissions`. - -The ``dump`` command supports the following options: - -``-a``, ``--all-threads`` - Dump every thread in the target process. Without this flag only the main - thread is shown. - -``--native`` - Include synthetic ```` frames marking transitions into C - extensions or other non-Python code. - -``--no-gc`` - Hide the synthetic ```` frames that mark active garbage collection. - -``--opcodes`` - Annotate each frame with the bytecode opcode the thread is currently - executing (for example, ``opcode=CALL_KW``). Useful for - instruction-level investigation, including identifying specializations - chosen by the adaptive interpreter. - -``--async-aware`` - Reconstruct stacks across ``await`` boundaries. ``dump`` walks the task - graph and emits one section per task, with ```` markers separating - coroutines awaiting each other. - -``--async-mode {running,all}`` - Controls which tasks are included when ``--async-aware`` is enabled. - ``running`` shows only the task currently executing on each thread; - ``all`` (the default for ``dump``) also includes tasks suspended on a - wait. ``attach``'s default for this flag is ``running``; ``dump`` - defaults to ``all`` because a single snapshot is most useful when it - shows the full task graph. - -``--blocking`` - Pause every thread in the target while reading its stack and resume - them after. Guarantees a fully consistent snapshot at the cost of - briefly stopping the target. Without it, ``dump`` reads memory while - the target keeps running, which is faster but can occasionally produce - a torn stack. - - .. _replay-command: The ``replay`` command @@ -1518,52 +1441,11 @@ Global options Attach to and profile a running process by PID. -.. option:: dump - - Print a single one-shot snapshot of a running process's Python stack. - .. option:: replay Convert a binary profile file to another output format. -Dump options ------------- - -The following options apply to the ``dump`` subcommand: - -.. option:: -a, --all-threads - - Dump all threads in the target process instead of just the main thread. - -.. option:: --native - - Include ```` frames for non-Python code. - -.. option:: --no-gc - - Exclude ```` frames for active garbage collection. - -.. option:: --opcodes - - Show bytecode opcode names when available. - -.. option:: --async-aware - - Reconstruct the stack across ``await`` boundaries for asyncio - applications. - -.. option:: --async-mode - - Async stack mode: ``running`` (only the running task) or ``all`` - (all tasks including waiting). Defaults to ``all`` for ``dump``. - Requires :option:`--async-aware`. - -.. option:: --blocking - - Pause all threads in the target process while reading the stack. - - Sampling options ---------------- diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 3703d2fa60056f..04895ae4ec524b 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -17,7 +17,7 @@ import can be suppressed using the interpreter's :option:`-S` option. Importing this module normally appends site-specific paths to the module search path and adds :ref:`callables `, including :func:`help` to the built-in -namespace. However, Python startup option :option:`-S` blocks this, and this module +namespace. However, Python startup option :option:`-S` blocks this and this module can be safely imported with no automatic modifications to the module search path or additions to the builtins. To explicitly trigger the usual site-specific additions, call the :func:`main` function. @@ -71,121 +71,40 @@ the user site prefixes are also implicitly not searched for site-packages. single: # (hash); comment pair: statement; import -The :mod:`!site` module recognizes two startup configuration files of the form -:file:`{name}.pth` for path configurations, and :file:`{name}.start` for -pre-first-line code execution. Both files can exist in one of the four -directories mentioned above. Within each directory, these files are sorted -alphabetically by filename, then parsed in sorted order. +A path configuration file is a file whose name has the form :file:`{name}.pth` +and exists in one of the four directories mentioned above; its contents are +additional items (one per line) to be added to ``sys.path``. Non-existing items +are never added to ``sys.path``, and no check is made that the item refers to a +directory rather than a file. No item is added to ``sys.path`` more than +once. Blank lines and lines beginning with ``#`` are skipped. Lines starting +with ``import`` (followed by space or tab) are executed. -.. _site-pth-files: - -Path extensions (:file:`.pth` files) ------------------------------------- - -:file:`{name}.pth` contains additional items (one per line) to be appended to -``sys.path``. Items that name non-existing directories are never added to -``sys.path``, and no check is made that the item refers to a directory rather -than a file. No item is added to ``sys.path`` more than once. Blank lines -and lines beginning with ``#`` are skipped. +.. note:: -For backward compatibility, lines starting with ``import`` (followed by space -or tab) are executed with :func:`exec`. + An executable line in a :file:`.pth` file is run at every Python startup, + regardless of whether a particular module is actually going to be used. + Its impact should thus be kept to a minimum. + The primary intended purpose of executable lines is to make the + corresponding module(s) importable + (load 3rd-party import hooks, adjust :envvar:`PATH` etc). + Any other initialization is supposed to be done upon a module's + actual import, if and when it happens. + Limiting a code chunk to a single line is a deliberate measure + to discourage putting anything more complex here. .. versionchanged:: 3.13 - The :file:`.pth` files are now decoded by UTF-8 at first and then by the :term:`locale encoding` if it fails. -.. versionchanged:: next - - :file:`.pth` file lines starting with ``import`` are deprecated. During - the deprecation period, such lines are still executed (except in the case - below), but a diagnostic message is emitted only when the :option:`-v` flag - is given. - - ``import`` lines in :file:`{name}.pth` are silently ignored when a - :ref:`matching ` :file:`{name}.start` file exists. - - Errors on individual lines no longer abort processing of the rest of the - file. Each error is reported and the remaining lines continue to be - processed. - -.. deprecated-removed:: next 3.20 - - Decoding :file:`{name}.pth` files in any encoding other than ``utf-8-sig`` - is deprecated in Python 3.15, and support for decoding from the locale - encoding will be removed in Python 3.20. - - ``import`` lines in :file:`{name}.pth` files are deprecated and will be - silently ignored in Python 3.18 and 3.19. In Python 3.20 a warning will be - produced for ``import`` lines in :file:`{name}.pth` files. - - -.. _site-start-files: - -Startup entry points (:file:`.start` files) -------------------------------------------- - -.. versionadded:: next - -A startup entry point file is a file whose name has the form -:file:`{name}.start` and exists in one of the site-packages directories -described above. Each file specifies entry points to be called during -interpreter startup, using the ``pkg.mod:callable`` syntax understood by -:func:`pkgutil.resolve_name`. - -Each non-blank line that does not begin with ``#`` must contain an entry -point reference in the form ``pkg.mod:callable``. The colon and callable -portion are mandatory. Each callable is invoked with no arguments, and -any return value is discarded. - -:file:`.start` files are processed after all :file:`.pth` path extensions -have been applied to :data:`sys.path`, ensuring that paths are available -before any startup code runs. - -Unlike :data:`sys.path` extensions from :file:`.pth` files, duplicate entry -points are **not** de-duplicated --- if an entry point appears more than once, -it will be called more than once. - -If an exception occurs during resolution or invocation of an entry point, -a traceback is printed to :data:`sys.stderr` and processing continues with -the remaining entry points. - -:file:`.start` files must be encoded in UTF-8. - -:pep:`829` defined the original specification for these features. - -.. note:: - - If a :file:`{name}.start` file exists alongside a :file:`{name}.pth` file - with the same base name, any ``import`` lines in the :file:`.pth` file are - ignored in favor of the entry points in the :file:`.start` file. - -.. note:: - - Executable lines (``import`` lines in :file:`{name}.pth` files and - :file:`{name}.start` file entry points) are always run at Python startup - (unless :option:`-S` is given to disable the ``site.py`` module entirely), - regardless of whether a particular module is actually going to be used. - -.. note:: - - :file:`{name}.start` files invoke :func:`pkgutil.resolve_name` with - ``strict=True``, which requires the full ``pkg.mod:callable`` form. - .. index:: single: package triple: path; configuration; file - -Startup file examples ---------------------- - For example, suppose ``sys.prefix`` and ``sys.exec_prefix`` are set to :file:`/usr/local`. The Python X.Y library is then installed in :file:`/usr/local/lib/python{X.Y}`. Suppose this has a subdirectory :file:`/usr/local/lib/python{X.Y}/site-packages` with three -sub-subdirectories, :file:`foo`, :file:`bar` and :file:`spam`, and two path +subsubdirectories, :file:`foo`, :file:`bar` and :file:`spam`, and two path configuration files, :file:`foo.pth` and :file:`bar.pth`. Assume :file:`foo.pth` contains the following:: @@ -212,45 +131,6 @@ directory precedes the :file:`foo` directory because :file:`bar.pth` comes alphabetically before :file:`foo.pth`; and :file:`spam` is omitted because it is not mentioned in either path configuration file. -Let's say that there is also a :file:`foo.start` file containing the -following:: - - # foo package startup code - - foo.submod:initialize - -Now, after ``sys.path`` has been extended as above, and before Python turns -control over to user code, the ``foo.submod`` module is imported and the -``initialize()`` function from that module is called. - - -.. _site-migration-guide: - -Migrating from ``import`` lines in ``.pth`` files to ``.start`` files ---------------------------------------------------------------------- - -If your package currently ships a :file:`{name}.pth` file, you can keep all -``sys.path`` extension lines unchanged. Only ``import`` lines need to be -migrated. - -To migrate, create a callable (taking zero arguments) within an importable -module in your package. Reference it as a ``pkg.mod:callable`` entry point -in a matching :file:`{name}.start` file. Move everything on your ``import`` -line after the first semi-colon into the ``callable()`` function. - -If your package must straddle older Pythons that do not support :pep:`829` -and newer Pythons that do, change the ``import`` lines in your -:file:`{name}.pth` to use the following form: - -.. code-block:: python - - import pkg.mod; pkg.mod.callable() - -Older Pythons will execute these ``import`` lines, while newer Pythons will -ignore them in favor of the :file:`{name}.start` file. After the straddling -period, remove all ``import`` lines from your :file:`.pth` files. - - :mod:`!sitecustomize` --------------------- @@ -356,27 +236,10 @@ Module contents This function used to be called unconditionally. -.. function:: addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False) - - Add a directory to sys.path and parse the :file:`.pth` and :file:`.start` - files found in that directory. Typically used in :mod:`sitecustomize` or - :mod:`usercustomize` (see above). - - The *known_paths* argument is an optional set of case-normalized paths - used to prevent duplicate :data:`sys.path` entries. When ``None`` (the - default), the set is built from the current :data:`sys.path`. - - While :file:`.pth` and :file:`.start` files are always parsed, set - *defer_processing_start_files* to ``True`` to prevent processing the - startup data found in those files, so that you can process them explicitly - (this is typically used by the :func:`main` function). - - .. versionchanged:: next +.. function:: addsitedir(sitedir, known_paths=None) - Also processes :file:`.start` files. See :ref:`site-start-files`. - All :file:`.pth` and :file:`.start` files are now read and - accumulated before any path extensions, ``import`` line execution, - or entry point invocations take place. + Add a directory to sys.path and process its :file:`.pth` files. Typically + used in :mod:`sitecustomize` or :mod:`usercustomize` (see above). .. function:: getsitepackages() @@ -445,6 +308,5 @@ value greater than 2 if there is an error. .. seealso:: * :pep:`370` -- Per user site-packages directory - * :pep:`829` -- Startup entry points and the deprecation of import lines in ``.pth`` files * :ref:`sys-path-init` -- The initialization of :data:`sys.path`. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index f45a22addbb56a..17cf57dd00b4bd 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2156,7 +2156,7 @@ without the dedicated syntax, as documented below. Added support for the ``bound``, ``covariant``, ``contravariant``, and ``infer_variance`` parameters. -.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False, infer_variance=False, default=typing.NoDefault) +.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False, default=typing.NoDefault) Parameter specification variable. A specialized version of :ref:`type variables `. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 586a1306d83c4c..61a440d2ad6f8d 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -91,7 +91,6 @@ Summary -- Release highlights * :ref:`Improved error messages ` * :ref:`The official Windows 64-bit binaries now use the tail-calling interpreter ` -* :pep:`829`: Package Startup Configuration Files New features ============ @@ -322,9 +321,6 @@ Key features include: * Profile running processes by PID (``attach``) - attach to already-running applications * Run and profile scripts directly (``run``) - profile from the very start of execution * Execute and profile modules (``run -m``) - profile packages run as ``python -m module`` - * Capture a one-shot snapshot of a running process (``dump``) - print a - traceback-style stack of every thread (or all asyncio tasks with - ``--async-aware``). Useful for investigating hung processes. * **Multiple profiling modes**: Choose what to measure based on your performance investigation: diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index c2c508c1a71c5c..78d082306c2e33 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -285,6 +285,10 @@ Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); extern int _PyType_CheckConsistency(PyTypeObject *type); extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); +extern const PyNumberMethods _Py_empty_number_methods; +extern const PySequenceMethods _Py_empty_sequence_methods; +extern const PyMappingMethods _Py_empty_mapping_methods; + // Fast inlined version of PyType_HasFeature() static inline int _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 7c500e557f0180..62806b1d8d7bcf 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -359,23 +359,6 @@ class LiveProfiler(ThemeSection): ) -@dataclass(frozen=True, kw_only=True) -class ProfilerDump(ThemeSection): - header: str = ANSIColors.BOLD_BLUE - interpreter: str = ANSIColors.GREY - thread: str = ANSIColors.BOLD_CYAN - status: str = ANSIColors.YELLOW - frame_index: str = ANSIColors.GREY - frame: str = ANSIColors.BOLD_GREEN - filename: str = ANSIColors.CYAN - line_no: str = ANSIColors.YELLOW - source: str = ANSIColors.WHITE - source_highlight: str = ANSIColors.BOLD_YELLOW - opcode: str = ANSIColors.GREY - warning: str = ANSIColors.YELLOW - reset: str = ANSIColors.RESET - - @dataclass(frozen=True, kw_only=True) class Pickletools(ThemeSection): annotation: str = ANSIColors.GREY @@ -464,7 +447,6 @@ class Theme: http_server: HttpServer = field(default_factory=HttpServer) live_profiler: LiveProfiler = field(default_factory=LiveProfiler) pickletools: Pickletools = field(default_factory=Pickletools) - profiler_dump: ProfilerDump = field(default_factory=ProfilerDump) syntax: Syntax = field(default_factory=Syntax) timeit: Timeit = field(default_factory=Timeit) tokenize: Tokenize = field(default_factory=Tokenize) @@ -481,7 +463,6 @@ def copy_with( http_server: HttpServer | None = None, live_profiler: LiveProfiler | None = None, pickletools: Pickletools | None = None, - profiler_dump: ProfilerDump | None = None, syntax: Syntax | None = None, timeit: Timeit | None = None, tokenize: Tokenize | None = None, @@ -501,7 +482,6 @@ def copy_with( http_server=http_server or self.http_server, live_profiler=live_profiler or self.live_profiler, pickletools=pickletools or self.pickletools, - profiler_dump=profiler_dump or self.profiler_dump, syntax=syntax or self.syntax, timeit=timeit or self.timeit, tokenize=tokenize or self.tokenize, @@ -525,7 +505,6 @@ def no_colors(cls) -> Self: http_server=HttpServer.no_colors(), live_profiler=LiveProfiler.no_colors(), pickletools=Pickletools.no_colors(), - profiler_dump=ProfilerDump.no_colors(), syntax=Syntax.no_colors(), timeit=Timeit.no_colors(), tokenize=Tokenize.no_colors(), diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index d67cc4dd1b19ab..e9810d6bd5d57b 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -757,16 +757,22 @@ def _is_kw_only(a_type, dataclasses): return a_type is dataclasses.KW_ONLY -def _get_type_from_annotation(annotation, cls): - # Loosely parse a string annotation and return its type. +def _is_type(annotation, cls, a_module, a_type, is_type_predicate): + # Given a type annotation string, does it refer to a_type in + # a_module? For example, when checking that annotation denotes a + # ClassVar, then a_module is typing, and a_type is + # typing.ClassVar. - # We can't perform a full type hint evaluation at the point where @dataclass - # was invoked because class's module is not fully initialized yet. So we resort - # to parsing string annotation using regexp, and extracting a type before - # the first square bracket. + # It's possible to look up a_module given a_type, but it involves + # looking in sys.modules (again!), and seems like a waste since + # the caller already knows a_module. # - annotation is a string type annotation # - cls is the class that this annotation was found in + # - a_module is the module we want to match + # - a_type is the type in that module we want to match + # - is_type_predicate is a function called with (obj, a_module) + # that determines if obj is of the desired type. # Since this test does not do a local namespace lookup (and # instead only a module (global) lookup), there are some things it @@ -797,21 +803,24 @@ def _get_type_from_annotation(annotation, cls): # https://github.com/python/cpython/issues/77634 for details. global _MODULE_IDENTIFIER_RE if _MODULE_IDENTIFIER_RE is None: - _MODULE_IDENTIFIER_RE = re.compile(r'^\s*(\w+(?:\s*\.\s*\w+)*)') + _MODULE_IDENTIFIER_RE = re.compile(r'(?:\s*(\w+)\s*\.)?\s*(\w+)') match = _MODULE_IDENTIFIER_RE.prefixmatch(annotation) - if not match: - return None - - # Note: _MODULE_IDENTIFIER_RE guarantees that path is non-empty - path = match[1].split(".") - root = sys.modules.get(cls.__module__) - for path_item in path: - root = getattr(root, path_item.strip(), None) - if root is None: - return None - - return root + if match: + ns = None + module_name = match[1] + if not module_name: + # No module name, assume the class's module did + # "from dataclasses import InitVar". + ns = sys.modules.get(cls.__module__).__dict__ + else: + # Look up module_name in the class's module. + module = sys.modules.get(cls.__module__) + if module and module.__dict__.get(module_name) is a_module: + ns = sys.modules.get(a_type.__module__).__dict__ + if ns and is_type_predicate(ns.get(match[2]), a_module): + return True + return False def _get_field(cls, a_name, a_type, default_kw_only): @@ -849,10 +858,6 @@ def _get_field(cls, a_name, a_type, default_kw_only): # is actually of the correct type. # For the complete discussion, see https://bugs.python.org/issue33453 - if isinstance(a_type, str): - a_type_annotation = _get_type_from_annotation(a_type, cls) - else: - a_type_annotation = a_type # If typing has not been imported, then it's impossible for any # annotation to be a ClassVar. So, only look for ClassVar if @@ -860,7 +865,10 @@ def _get_field(cls, a_name, a_type, default_kw_only): # module). typing = sys.modules.get('typing') if typing: - if _is_classvar(a_type_annotation, typing): + if (_is_classvar(a_type, typing) + or (isinstance(f.type, str) + and _is_type(f.type, cls, typing, typing.ClassVar, + _is_classvar))): f._field_type = _FIELD_CLASSVAR # If the type is InitVar, or if it's a matching string annotation, @@ -869,7 +877,10 @@ def _get_field(cls, a_name, a_type, default_kw_only): # The module we're checking against is the module we're # currently in (dataclasses.py). dataclasses = sys.modules[__name__] - if _is_initvar(a_type_annotation, dataclasses): + if (_is_initvar(a_type, dataclasses) + or (isinstance(f.type, str) + and _is_type(f.type, cls, dataclasses, dataclasses.InitVar, + _is_initvar))): f._field_type = _FIELD_INITVAR # Validations for individual fields. This is delayed until now, @@ -1062,11 +1073,10 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, dataclasses = sys.modules[__name__] for name, type in cls_annotations.items(): # See if this is a marker to change the value of kw_only. - if isinstance(type, str): - a_type_annotation = _get_type_from_annotation(type, cls) - else: - a_type_annotation = type - if _is_kw_only(a_type_annotation, dataclasses): + if (_is_kw_only(type, dataclasses) + or (isinstance(type, str) + and _is_type(type, cls, dataclasses, dataclasses.KW_ONLY, + _is_kw_only))): # Switch the default to kw_only=True, and ignore this # annotation: it's not a real field. if KW_ONLY_seen: diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index 11c2a4b0ef4635..8772a66791a3c9 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -9,9 +9,6 @@ import os.path import sys -lazy import re - - __all__ = [ 'get_importer', 'iter_importers', 'walk_packages', 'iter_modules', 'get_data', @@ -401,10 +398,9 @@ def get_data(package, resource): return loader.get_data(resource_name) -_LENIENT_PATTERN = None -_STRICT_PATTERN = None +_NAME_PATTERN = None -def resolve_name(name, *, strict=False): +def resolve_name(name): """ Resolve a name to an object. @@ -414,7 +410,6 @@ def resolve_name(name, *, strict=False): W(.W)* W(.W)*:(W(.W)*)? - W(.W)*:(W(.W)*) The first form is intended for backward compatibility only. It assumes that some part of the dotted name is a package, and the rest is an object @@ -429,11 +424,6 @@ def resolve_name(name, *, strict=False): hierarchy within that package. Only one import is needed in this form. If it ends with the colon, then a module object is returned. - The first two forms are accepted when `strict=False` (the default). - - The third form requires both the module name and callable, separated by - a colon. Only this form is accepted when `strict=True`. - The function will return an object (which might be a module), or raise one of the following exceptions: @@ -442,26 +432,18 @@ def resolve_name(name, *, strict=False): AttributeError - if a failure occurred when traversing the object hierarchy within the imported package to get to the desired object. """ - global _LENIENT_PATTERN, _STRICT_PATTERN - dotted_words = r'(?!\d)(\w+)(\.(?!\d)(\w+))*' - if strict: - if _STRICT_PATTERN is None: - _STRICT_PATTERN = re.compile( - f'^(?P{dotted_words})' - f'(?P:(?P{dotted_words}))$', - re.UNICODE) - pattern = _STRICT_PATTERN - else: - if _LENIENT_PATTERN is None: - _LENIENT_PATTERN = re.compile( - f'^(?P{dotted_words})' - f'(?P:(?P{dotted_words})?)?$', - re.UNICODE) - pattern = _LENIENT_PATTERN - - if (m := pattern.match(name)) is None: + global _NAME_PATTERN + if _NAME_PATTERN is None: + # Lazy import to speedup Python startup time + import re + dotted_words = r'(?!\d)(\w+)(\.(?!\d)(\w+))*' + _NAME_PATTERN = re.compile(f'^(?P{dotted_words})' + f'(?P:(?P{dotted_words})?)?$', + re.UNICODE) + + m = _NAME_PATTERN.match(name) + if not m: raise ValueError(f'invalid format: {name!r}') - gd = m.groupdict() if gd.get('cln'): # there is a colon - a one-step import is all that's needed diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index bc879c43e15965..c0aa3ae024a120 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -14,8 +14,7 @@ from contextlib import nullcontext from .errors import SamplingUnknownProcessError, SamplingModuleNotFoundError, SamplingScriptNotFoundError -from .sample import sample, sample_live, dump_stack, _is_process_running -from .dump import print_stack_dump +from .sample import sample, sample_live, _is_process_running from .pstats_collector import PstatsCollector from .stack_collector import CollapsedStackCollector, FlamegraphCollector, DiffFlamegraphCollector from .heatmap_collector import HeatmapCollector @@ -73,9 +72,6 @@ def __call__(self, parser, namespace, values, option_string=None): # Attach to a running process `python -m profiling.sampling attach 1234` - # Dump a running process's current stack - `python -m profiling.sampling dump 1234` - # Live interactive mode for a script `python -m profiling.sampling run --live script.py` @@ -552,51 +548,6 @@ def _add_pstats_options(parser): ) -def _add_dump_options(parser): - """Add one-shot stack dump options to a parser.""" - dump_group = parser.add_argument_group("Dump options") - dump_group.add_argument( - "-a", - "--all-threads", - action="store_true", - help="Dump all threads in the process instead of just the main thread", - ) - dump_group.add_argument( - "--native", - action="store_true", - help='Include artificial "" frames to denote calls to non-Python code', - ) - dump_group.add_argument( - "--no-gc", - action="store_false", - dest="gc", - help='Don\'t include artificial "" frames to denote active garbage collection', - ) - dump_group.add_argument( - "--opcodes", - action="store_true", - help="Show bytecode opcode names when available.", - ) - dump_group.add_argument( - "--async-aware", - action="store_true", - help="Enable async-aware stack reconstruction", - ) - dump_group.add_argument( - "--async-mode", - choices=["running", "all"], - default=argparse.SUPPRESS, - help='Async stack mode: "running" (only running task) ' - 'or "all" (all tasks including waiting, default for dump). ' - "Requires --async-aware", - ) - dump_group.add_argument( - "--blocking", - action="store_true", - help="Stop all threads in target process before dumping the stack.", - ) - - def _sort_to_mode(sort_choice): """Convert sort choice string to SORT_MODE constant.""" sort_map = { @@ -834,11 +785,19 @@ def _validate_args(args, parser): args: Parsed command-line arguments parser: ArgumentParser instance for error reporting """ - command = getattr(args, 'command', None) - - if command == "replay": + # Replay command has no special validation needed + if getattr(args, 'command', None) == "replay": return + # Warn about blocking mode with aggressive sampling intervals + if args.blocking and args.sample_interval_usec < 100: + print( + f"Warning: --blocking with a {args.sample_interval_usec} µs interval will stop all threads " + f"{1_000_000 // args.sample_interval_usec} times per second. " + "Consider using --sampling-rate 1khz or lower to reduce overhead.", + file=sys.stderr + ) + # Check if live mode is available if hasattr(args, 'live') and args.live and LiveStatsCollector is None: parser.error( @@ -858,9 +817,9 @@ def _validate_args(args, parser): # Async-aware mode is incompatible with --native, --no-gc, --mode, and --all-threads if getattr(args, 'async_aware', False): issues = [] - if getattr(args, 'native', False): + if args.native: issues.append("--native") - if not getattr(args, 'gc', True): + if not args.gc: issues.append("--no-gc") if hasattr(args, 'mode') and args.mode != "wall": issues.append(f"--mode={args.mode}") @@ -872,26 +831,9 @@ def _validate_args(args, parser): "Async-aware profiling uses task-based stack reconstruction." ) - # --async-mode requires --async-aware when explicitly set - if not getattr(args, 'async_aware', False): - if command == "dump": - # dump uses SUPPRESS default, so attr only exists if user passed it - if hasattr(args, 'async_mode'): - parser.error("--async-mode requires --async-aware to be enabled.") - elif hasattr(args, 'async_mode') and args.async_mode != "running": - parser.error("--async-mode requires --async-aware to be enabled.") - - if command == "dump": - return - - # Warn about blocking mode with aggressive sampling intervals - if args.blocking and args.sample_interval_usec < 100: - print( - f"Warning: --blocking with a {args.sample_interval_usec} µs interval will stop all threads " - f"{1_000_000 // args.sample_interval_usec} times per second. " - "Consider using --sampling-rate 1khz or lower to reduce overhead.", - file=sys.stderr - ) + # --async-mode requires --async-aware + if hasattr(args, 'async_mode') and args.async_mode != "running" and not getattr(args, 'async_aware', False): + parser.error("--async-mode requires --async-aware to be enabled.") # Live mode is incompatible with format options if hasattr(args, 'live') and args.live: @@ -1048,27 +990,6 @@ def _main(): _add_format_options(attach_parser) _add_pstats_options(attach_parser) - # === DUMP COMMAND === - dump_parser = subparsers.add_parser( - "dump", - help="Dump a running process's current stack", - formatter_class=CustomFormatter, - description="""Dump a running process's current Python stack - -Examples: - # Dump the main thread stack - `python -m profiling.sampling dump 1234` - - # Dump all thread stacks - `python -m profiling.sampling dump -a 1234`""", - ) - dump_parser.add_argument( - "pid", - type=int, - help="Process ID to dump", - ) - _add_dump_options(dump_parser) - # === REPLAY COMMAND === replay_parser = subparsers.add_parser( "replay", @@ -1103,7 +1024,6 @@ def _main(): command_handlers = { "run": _handle_run, "attach": _handle_attach, - "dump": _handle_dump, "replay": _handle_replay, } @@ -1165,34 +1085,6 @@ def _handle_attach(args): _handle_output(collector, args, args.pid, mode) -def _handle_dump(args): - if not _is_process_running(args.pid): - raise SamplingUnknownProcessError(args.pid) - - # Async-aware reconstruction requires wall mode so every thread is sampled, - # not just those holding the GIL or on CPU. - mode = PROFILING_MODE_WALL if args.async_aware else PROFILING_MODE_ALL - async_mode = getattr(args, "async_mode", "all") if args.async_aware else None - try: - stack_frames = dump_stack( - args.pid, - all_threads=args.all_threads, - mode=mode, - async_aware=async_mode, - native=args.native, - gc=args.gc, - opcodes=args.opcodes, - blocking=args.blocking, - ) - except ProcessLookupError: - sys.exit( - f"No stack dump collected - process {args.pid} exited before " - "its stack could be read." - ) - - print_stack_dump(stack_frames, pid=args.pid) - - def _handle_run(args): """Handle the 'run' command.""" # Validate target exists before launching subprocess diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py index 08759b611696b7..7dc095c6c279bd 100644 --- a/Lib/profiling/sampling/collector.py +++ b/Lib/profiling/sampling/collector.py @@ -62,80 +62,6 @@ def filter_internal_frames(frames): return [f for f in frames if not _is_internal_frame(f)] -def _build_task_graph(awaited_info_list): - task_map = {} - child_to_parent = {} # child_id -> (selected_parent_id, parent_count) - all_task_ids = set() - all_parent_ids = set() - - for awaited_info in awaited_info_list: - thread_id = awaited_info.thread_id - for task_info in awaited_info.awaited_by: - task_id = task_info.task_id - task_map[task_id] = (task_info, thread_id) - all_task_ids.add(task_id) - - if task_info.awaited_by: - parent_ids = [p.task_name for p in task_info.awaited_by] - parent_count = len(parent_ids) - all_parent_ids.update(parent_ids) - selected_parent = min(parent_ids) if parent_count > 1 else parent_ids[0] - child_to_parent[task_id] = (selected_parent, parent_count) - - return task_map, child_to_parent, all_task_ids, all_parent_ids - - -def _find_leaf_tasks(all_task_ids, all_parent_ids): - return all_task_ids - all_parent_ids - - -def _build_linear_stacks(leaf_task_ids, task_map, child_to_parent): - for leaf_id in leaf_task_ids: - frames = [] - visited = set() - current_id = leaf_id - thread_id = None - - while current_id is not None: - if current_id in visited: - break - visited.add(current_id) - - if current_id not in task_map: - break - - task_info, tid = task_map[current_id] - - if thread_id is None: - thread_id = tid - - if task_info.coroutine_stack: - for coro_info in task_info.coroutine_stack: - for frame in coro_info.call_stack: - frames.append(frame) - - parent_info = child_to_parent.get(current_id) - task_name = task_info.task_name or "Task-" + str(task_info.task_id) - if parent_info: - selected_parent, parent_count = parent_info - if parent_count > 1: - task_name = f"{task_name} ({parent_count} parents)" - frames.append(FrameInfo(("", None, task_name, None))) - current_id = selected_parent - else: - frames.append(FrameInfo(("", None, task_name, None))) - current_id = None - - if frames and thread_id is not None: - yield frames, thread_id, leaf_id - - -def iter_async_frames(awaited_info_list): - task_map, child_to_parent, all_task_ids, all_parent_ids = _build_task_graph(awaited_info_list) - leaf_task_ids = _find_leaf_tasks(all_task_ids, all_parent_ids) - yield from _build_linear_stacks(leaf_task_ids, task_map, child_to_parent) - - class Collector(ABC): @abstractmethod def collect(self, stack_frames, timestamps_us=None): @@ -180,12 +106,19 @@ def _iter_all_frames(self, stack_frames, skip_idle=False): yield frames, thread_info.thread_id def _iter_async_frames(self, awaited_info_list): - yield from iter_async_frames(awaited_info_list) + # Phase 1: Index tasks and build parent relationships with pre-computed selection + task_map, child_to_parent, all_task_ids, all_parent_ids = self._build_task_graph(awaited_info_list) + + # Phase 2: Find leaf tasks (tasks not awaited by anyone) + leaf_task_ids = self._find_leaf_tasks(all_task_ids, all_parent_ids) + + # Phase 3: Build linear stacks from each leaf to root (optimized - no sorting!) + yield from self._build_linear_stacks(leaf_task_ids, task_map, child_to_parent) def _iter_stacks(self, stack_frames, skip_idle=False): """Yield (frames, thread_id) for all stacks, handling both sync and async modes.""" if stack_frames and hasattr(stack_frames[0], "awaited_by"): - for frames, thread_id, _ in iter_async_frames(stack_frames): + for frames, thread_id, _ in self._iter_async_frames(stack_frames): if frames: yield frames, thread_id else: @@ -193,6 +126,85 @@ def _iter_stacks(self, stack_frames, skip_idle=False): if frames: yield frames, thread_id + def _build_task_graph(self, awaited_info_list): + task_map = {} + child_to_parent = {} # Maps child_id -> (selected_parent_id, parent_count) + all_task_ids = set() + all_parent_ids = set() # Track ALL parent IDs for leaf detection + + for awaited_info in awaited_info_list: + thread_id = awaited_info.thread_id + for task_info in awaited_info.awaited_by: + task_id = task_info.task_id + task_map[task_id] = (task_info, thread_id) + all_task_ids.add(task_id) + + # Pre-compute selected parent and count for optimization + if task_info.awaited_by: + parent_ids = [p.task_name for p in task_info.awaited_by] + parent_count = len(parent_ids) + # Track ALL parents for leaf detection + all_parent_ids.update(parent_ids) + # Use min() for O(n) instead of sorted()[0] which is O(n log n) + selected_parent = min(parent_ids) if parent_count > 1 else parent_ids[0] + child_to_parent[task_id] = (selected_parent, parent_count) + + return task_map, child_to_parent, all_task_ids, all_parent_ids + + def _find_leaf_tasks(self, all_task_ids, all_parent_ids): + # Leaves are tasks that are not parents of any other task + return all_task_ids - all_parent_ids + + def _build_linear_stacks(self, leaf_task_ids, task_map, child_to_parent): + for leaf_id in leaf_task_ids: + frames = [] + visited = set() + current_id = leaf_id + thread_id = None + + # Follow the single parent chain from leaf to root + while current_id is not None: + # Cycle detection + if current_id in visited: + break + visited.add(current_id) + + # Check if task exists in task_map + if current_id not in task_map: + break + + task_info, tid = task_map[current_id] + + # Set thread_id from first task + if thread_id is None: + thread_id = tid + + # Add all frames from all coroutines in this task + if task_info.coroutine_stack: + for coro_info in task_info.coroutine_stack: + for frame in coro_info.call_stack: + frames.append(frame) + + # Get pre-computed parent info (no sorting needed!) + parent_info = child_to_parent.get(current_id) + + # Add task boundary marker with parent count annotation if multiple parents + task_name = task_info.task_name or "Task-" + str(task_info.task_id) + if parent_info: + selected_parent, parent_count = parent_info + if parent_count > 1: + task_name = f"{task_name} ({parent_count} parents)" + frames.append(FrameInfo(("", None, task_name, None))) + current_id = selected_parent + else: + # Root task - no parent + frames.append(FrameInfo(("", None, task_name, None))) + current_id = None + + # Yield the complete stack if we collected any frames + if frames and thread_id is not None: + yield frames, thread_id, leaf_id + def _is_gc_frame(self, frame): if isinstance(frame, tuple): funcname = frame[2] if len(frame) >= 3 else "" diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 9e315c080c353d..9195f5ee6dd390 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -87,18 +87,6 @@ def _new_unwinder(self, native, gc, opcodes, skip_non_matching_threads): **kwargs ) - def _get_stack_trace(self, async_aware=None): - with _pause_threads(self.unwinder, self.blocking): - if async_aware == "all": - return self.unwinder.get_all_awaited_by() - if async_aware == "running": - return self.unwinder.get_async_stack_trace() - return self.unwinder.get_stack_trace() - - def dump_stack(self, *, async_aware=None): - """Return a single stack snapshot from the target process.""" - return self._get_stack_trace(async_aware=async_aware) - def sample(self, collector, duration_sec=None, *, async_aware=False): sample_interval_sec = self.sample_interval_usec / 1_000_000 num_samples = 0 @@ -122,10 +110,14 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): time.sleep(sleep_time) elif next_time < current_time: try: - stack_frames = self._get_stack_trace( - async_aware=async_aware - ) - collector.collect(stack_frames) + with _pause_threads(self.unwinder, self.blocking): + if async_aware == "all": + stack_frames = self.unwinder.get_all_awaited_by() + elif async_aware == "running": + stack_frames = self.unwinder.get_async_stack_trace() + else: + stack_frames = self.unwinder.get_stack_trace() + collector.collect(stack_frames) except ProcessLookupError as e: running_time_sec = current_time - start_time break @@ -448,37 +440,6 @@ def sample( return collector -def dump_stack( - pid, - *, - all_threads=False, - mode=PROFILING_MODE_ALL, - async_aware=None, - native=False, - gc=True, - opcodes=False, - blocking=False, -): - """Return a single stack snapshot from a process.""" - if mode == PROFILING_MODE_ALL: - skip_non_matching_threads = False - else: - skip_non_matching_threads = True - - profiler = SampleProfiler( - pid, - sample_interval_usec=1, - all_threads=all_threads, - mode=mode, - native=native, - gc=gc, - opcodes=opcodes, - skip_non_matching_threads=skip_non_matching_threads, - blocking=blocking, - ) - return profiler.dump_stack(async_aware=async_aware) - - def sample_live( pid, collector, diff --git a/Lib/site.py b/Lib/site.py index 52dd9648734c3e..30015b3f26b4b3 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -18,26 +18,57 @@ it is also checked for site-packages (sys.base_prefix and sys.base_exec_prefix will always be the "real" prefixes of the Python installation). If "pyvenv.cfg" (a bootstrap configuration file) contains -the key "include-system-site-packages" set to "true" (case-insensitive), -the system-level prefixes will still also be searched for site-packages; -otherwise they won't. - -Two kinds of configuration files are processed in each site-packages -directory: - -- .pth files extend sys.path with additional directories (one per - line). Lines starting with "import" are deprecated (see PEP 829). - -- .start files specify startup entry points using the pkg.mod:callable - syntax. These are resolved via pkgutil.resolve_name() and called with no - arguments. - -When called from main(), all .pth path extensions are applied before any -.start entry points are executed, ensuring that paths are available before -startup code runs. - -See the documentation for the site module for full details: -https://docs.python.org/3/library/site.html +the key "include-system-site-packages" is set to "true" +(case-insensitive), the system-level prefixes will still also be +searched for site-packages; otherwise they won't. If the system-level +prefixes are not included then the user site prefixes are also implicitly +not searched for site-packages. + +All of the resulting site-specific directories, if they exist, are +appended to sys.path, and also inspected for path configuration +files. + +A path configuration file is a file whose name has the form +.pth; its contents are additional directories (one per line) +to be added to sys.path. Non-existing directories (or +non-directories) are never added to sys.path; no directory is added to +sys.path more than once. Blank lines and lines beginning with +'#' are skipped. Lines starting with 'import' are executed. + +For example, suppose sys.prefix and sys.exec_prefix are set to +/usr/local and there is a directory /usr/local/lib/python2.5/site-packages +with three subdirectories, foo, bar and spam, and two path +configuration files, foo.pth and bar.pth. Assume foo.pth contains the +following: + + # foo package configuration + foo + bar + bletch + +and bar.pth contains: + + # bar package configuration + bar + +Then the following directories are added to sys.path, in this order: + + /usr/local/lib/python2.5/site-packages/bar + /usr/local/lib/python2.5/site-packages/foo + +Note that bletch is omitted because it doesn't exist; bar precedes foo +because bar.pth comes alphabetically before foo.pth; and spam is +omitted because it is not mentioned in either path configuration file. + +The readline module is also automatically configured to enable +completion for systems that support it. This can be overridden in +sitecustomize, usercustomize or PYTHONSTARTUP. Starting Python in +isolated mode (-I) disables automatic readline configuration. + +After these operations, an attempt is made to import a module +named sitecustomize, which can perform arbitrary additional +site-specific customizations. If this import fails with an +ImportError exception, it is silently ignored. """ import sys @@ -48,11 +79,6 @@ import stat import errno -lazy import locale -lazy import pkgutil -lazy import traceback -lazy import warnings - # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] # Enable per user site-packages directory @@ -66,32 +92,15 @@ USER_BASE = None -def _trace(message, exc=None): +def _trace(message): if sys.flags.verbose: - _print_error(message, exc) - - -def _print_error(message, exc=None): - """Print an error message to stderr, optionally with a formatted traceback.""" - print(message, file=sys.stderr) - if exc is not None: - for record in traceback.format_exception(exc): - for line in record.splitlines(): - print(' ' + line, file=sys.stderr) + print(message, file=sys.stderr) def _warn(*args, **kwargs): - warnings.warn(*args, **kwargs) - + import warnings -def _warn_future_us(message, remove): - # Don't call warnings._deprecated() directly because we're lazily importing warnings and don't - # want to have to trigger an eager import if it's not necessary. Startup time matters a lot - # here and warnings isn't cheap! This inlines the check from - # warnings._py_warnings._deprecated(). - _version = sys.version_info - if (_version[:2] > remove) or (_version[:2] == remove and _version[3] != "alpha"): - warnings._deprecated(message, remove=remove) + warnings.warn(*args, **kwargs) def makepath(*paths): @@ -154,232 +163,75 @@ def _init_pathinfo(): return d -# Accumulated entry points from .start files across all site-packages -# directories. Execution is deferred until all paths in .pth files have been -# appended to sys.path. Map the .pth/.start file the data is found in to the -# data. -_pending_entrypoints = {} -_pending_syspaths = {} -_pending_importexecs = {} - - -def _read_pthstart_file(sitedir, name, suffix): - """Parse a .start or .pth file and return (lines, filename). - - On success, ``lines`` is a (possibly empty) list of the file's lines. - On failure (file missing, hidden, unreadable, or .start with bad - encoding), ``lines`` is ``None`` so callers can distinguish a - successfully-read empty file from one that could not be read. +def addpackage(sitedir, name, known_paths): + """Process a .pth file within the site-packages directory: + For each line in the file, either combine it with sitedir to a path + and add that to known_paths, or execute it if it starts with 'import '. """ - filename = os.path.join(sitedir, name) - _trace(f"Reading startup configuration file: {filename}") - + if known_paths is None: + known_paths = _init_pathinfo() + reset = True + else: + reset = False + fullname = os.path.join(sitedir, name) try: - st = os.lstat(filename) - except OSError as exc: - _trace(f"Cannot stat {filename!r}", exc) - return None, filename - + st = os.lstat(fullname) + except OSError: + return if ((getattr(st, 'st_flags', 0) & stat.UF_HIDDEN) or (getattr(st, 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_HIDDEN)): - _trace(f"Skipping hidden {suffix} file: {filename!r}") - return None, filename - - _trace(f"Processing {suffix} file: {filename!r}") + _trace(f"Skipping hidden .pth file: {fullname!r}") + return + _trace(f"Processing .pth file: {fullname!r}") try: - with io.open_code(filename) as f: - raw_content = f.read() - except OSError as exc: - _trace(f"Cannot read {filename!r}", exc) - return None, filename + with io.open_code(fullname) as f: + pth_content = f.read() + except OSError: + return try: - # Accept BOM markers in .start and .pth files as we do in source files (Windows PowerShell - # 5.1 makes it hard to emit UTF-8 files without a BOM). - content = raw_content.decode("utf-8-sig") + # Accept BOM markers in .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM) + pth_content = pth_content.decode("utf-8-sig") except UnicodeDecodeError: - _trace(f"Cannot read {filename!r} as UTF-8.") - # For .pth files only, and then only until Python 3.20, fallback to locale encoding for - # backward compatibility. - _warn_future_us( - ".pth files decoded to locale encoding as a fallback", - remove=(3, 20) - ) - if suffix == ".pth": - content = raw_content.decode(locale.getencoding()) - _trace(f"Using fallback encoding {locale.getencoding()!r}") - else: - return None, filename - - return content.splitlines(), filename - - -def _read_pth_file(sitedir, name, known_paths): - """Parse a .pth file, accumulating sys.path extensions and import lines. - - Errors on individual lines do not abort processing of the rest of the - file (PEP 829). - """ - lines, filename = _read_pthstart_file(sitedir, name, ".pth") - if lines is None: - return - - for n, line in enumerate(lines, 1): - line = line.strip() - if not line or line.startswith("#"): + # Fallback to locale encoding for backward compatibility. + # We will deprecate this fallback in the future. + import locale + pth_content = pth_content.decode(locale.getencoding()) + _trace(f"Cannot read {fullname!r} as UTF-8. " + f"Using fallback encoding {locale.getencoding()!r}") + + for n, line in enumerate(pth_content.splitlines(), 1): + if line.startswith("#"): continue - - # In Python 3.18 and 3.19, `import` lines are silently ignored. In - # Python 3.20 and beyond, issue a warning when `import` lines in .pth - # files are detected. - if line.startswith(("import ", "import\t")): - _warn_future_us( - "import lines in .pth files are silently ignored", - remove=(3, 18) - ) - _warn_future_us( - "import lines in .pth files are noisily ignored", - remove=(3, 20) - ) - _pending_importexecs.setdefault(filename, []).append(line) + if line.strip() == "": continue - try: - dir_, dircase = makepath(sitedir, line) - except Exception as exc: - _trace(f"Error in {filename!r}, line {n:d}: {line!r}", exc) - continue - - if dircase in known_paths: - _trace(f"In {filename!r}, line {n:d}: " - f"skipping duplicate sys.path entry: {dir_}") - else: - _pending_syspaths.setdefault(filename, []).append(dir_) - known_paths.add(dircase) - - -def _read_start_file(sitedir, name): - """Parse a .start file for a list of entry point strings.""" - lines, filename = _read_pthstart_file(sitedir, name, ".start") - if lines is None: - return - - # PEP 829: the *presence* of a matching .start file disables `import` - # line processing in the matched .pth file, regardless of whether the - # .start file produced any entry points. Register the filename as a - # key now so an empty (or comment-only) .start file still suppresses. - entrypoints = _pending_entrypoints.setdefault(filename, []) - - for n, line in enumerate(lines, 1): - line = line.strip() - if not line or line.startswith("#"): - continue - # Syntax validation is deferred to entry-point execution time, - # where pkgutil.resolve_name(strict=True) enforces the - # pkg.mod:callable form. - entrypoints.append(line) - - -def _extend_syspath(): - # We've already filtered out duplicates, either in the existing sys.path - # or in all the .pth files we've seen. We've also abspath/normpath'd all - # the entries, so all that's left to do is to ensure that the path exists. - for filename, dirs in _pending_syspaths.items(): - for dir_ in dirs: - if os.path.exists(dir_): - _trace(f"Extending sys.path with {dir_} from {filename}") - sys.path.append(dir_) - else: - _print_error( - f"In {filename}: {dir_} does not exist; " - f"skipping sys.path append") - - -def _exec_imports(): - # For all the `import` lines we've seen in .pth files, exec() them in - # order. However, if they come from a file with a matching .start, then - # we ignore these import lines. For the ones we do process, print a - # warning but only when -v was given. - for filename, imports in _pending_importexecs.items(): - name, dot, pth = filename.rpartition(".") - assert dot == "." and pth == "pth", f"Bad startup filename: {filename}" - - if f"{name}.start" in _pending_entrypoints: - # Skip import lines in favor of entry points. - continue - - _trace( - f"import lines in {filename} are deprecated, " - f"use entry points in a {name}.start file instead." - ) - - for line in imports: - try: - _trace(f"Exec'ing from {filename}: {line}") + if line.startswith(("import ", "import\t")): exec(line) - except Exception as exc: - _print_error( - f"Error in import line from {filename}: {line}", exc) - - -def _execute_start_entrypoints(): - """Execute all accumulated .start file entry points. - - Called after all site-packages directories have been processed so that - sys.path is fully populated before any entry point code runs. Uses - pkgutil.resolve_name(strict=True) which both validates the strict - pkg.mod:callable form and resolves the entry point in one step. - """ - for filename, entrypoints in _pending_entrypoints.items(): - for entrypoint in entrypoints: - try: - _trace(f"Executing entry point: {entrypoint} from {filename}") - callable_ = pkgutil.resolve_name(entrypoint, strict=True) - except ValueError as exc: - _print_error( - f"Invalid entry point syntax in {filename}: " - f"{entrypoint!r}", exc) - continue - except Exception as exc: - _print_error( - f"Error resolving entry point {entrypoint} " - f"from {filename}", exc) continue - try: - callable_() - except Exception as exc: - _print_error( - f"Error in entry point {entrypoint} from {filename}", - exc) - - -def process_startup_files(): - """Flush all pending sys.path and entry points.""" - _extend_syspath() - _exec_imports() - _execute_start_entrypoints() - _pending_syspaths.clear() - _pending_importexecs.clear() - _pending_entrypoints.clear() - - -def addpackage(sitedir, name, known_paths): - """Process a .pth file within the site-packages directory.""" - if known_paths is None: - known_paths = _init_pathinfo() - reset = True - else: - reset = False - _read_pth_file(sitedir, name, known_paths) - process_startup_files() + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if dircase not in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception as exc: + print(f"Error processing line {n:d} of {fullname}:\n", + file=sys.stderr) + import traceback + for record in traceback.format_exception(exc): + for line in record.splitlines(): + print(' '+line, file=sys.stderr) + print("\nRemainder of file ignored", file=sys.stderr) + break if reset: known_paths = None return known_paths -def addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False): - """Add 'sitedir' argument to sys.path if missing and handle startup - files.""" +def addsitedir(sitedir, known_paths=None): + """Add 'sitedir' argument to sys.path if missing and handle .pth files in + 'sitedir'""" _trace(f"Adding directory: {sitedir!r}") if known_paths is None: known_paths = _init_pathinfo() @@ -394,36 +246,12 @@ def addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False) names = os.listdir(sitedir) except OSError: return - - # The following phases are defined by PEP 829. - # Phases 1-3: Read .pth files, accumulating paths and import lines. - pth_names = sorted( - name for name in names - if name.endswith(".pth") and not name.startswith(".") - ) - for name in pth_names: - _read_pth_file(sitedir, name, known_paths) - - # Phases 6-7: Discover .start files and accumulate their entry points. - # Import lines from .pth files with a matching .start file are discarded - # at flush time by _exec_imports(). - start_names = sorted( - name for name in names - if name.endswith(".start") and not name.startswith(".") - ) - for name in start_names: - _read_start_file(sitedir, name) - - # Generally, when addsitedir() is called explicitly, we'll want to process - # all the startup file data immediately. However, when called through - # main(), we'll want to batch up all the startup file processing. main() - # will set this flag to True to defer processing. - if not defer_processing_start_files: - process_startup_files() - + names = [name for name in names + if name.endswith(".pth") and not name.startswith(".")] + for name in sorted(names): + addpackage(sitedir, name, known_paths) if reset: known_paths = None - return known_paths @@ -536,7 +364,7 @@ def getusersitepackages(): return USER_SITE -def addusersitepackages(known_paths, *, defer_processing_start_files=False): +def addusersitepackages(known_paths): """Add a per user site-package to sys.path Each user has its own python directory with site-packages in the @@ -548,7 +376,7 @@ def addusersitepackages(known_paths, *, defer_processing_start_files=False): user_site = getusersitepackages() if ENABLE_USER_SITE and os.path.isdir(user_site): - addsitedir(user_site, known_paths, defer_processing_start_files=defer_processing_start_files) + addsitedir(user_site, known_paths) return known_paths def getsitepackages(prefixes=None): @@ -590,12 +418,12 @@ def getsitepackages(prefixes=None): sitepackages.append(os.path.join(prefix, "Lib", "site-packages")) return sitepackages -def addsitepackages(known_paths, prefixes=None, *, defer_processing_start_files=False): +def addsitepackages(known_paths, prefixes=None): """Add site-packages to sys.path""" _trace("Processing global site-packages") for sitedir in getsitepackages(prefixes): if os.path.isdir(sitedir): - addsitedir(sitedir, known_paths, defer_processing_start_files=defer_processing_start_files) + addsitedir(sitedir, known_paths) return known_paths @@ -877,15 +705,8 @@ def main(): known_paths = venv(known_paths) if ENABLE_USER_SITE is None: ENABLE_USER_SITE = check_enableusersite() - known_paths = addusersitepackages(known_paths, defer_processing_start_files=True) - known_paths = addsitepackages(known_paths, defer_processing_start_files=True) - # PEP 829: flush accumulated data from all .pth and .start files. - # Paths are extended first, then deprecated import lines are exec'd, - # and finally .start entry points are executed — ensuring sys.path is - # fully populated before any startup code runs. process_startup_files() - # also clears the pending state so a later addsitedir() call does - # not re-apply already-processed data. - process_startup_files() + known_paths = addusersitepackages(known_paths) + known_paths = addsitepackages(known_paths) setquit() setcopyright() sethelper() diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index dcd6a3ef9abfab..8a0a7d12c04aa4 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -4353,17 +4353,10 @@ def test_classvar_module_level_import(self): from test.test_dataclasses import dataclass_module_1_str from test.test_dataclasses import dataclass_module_2 from test.test_dataclasses import dataclass_module_2_str - from test.test_dataclasses import dataclass_module_3 - from test.test_dataclasses import dataclass_module_3_str - from test.test_dataclasses import dataclass_module_4 - from test.test_dataclasses import dataclass_module_4_str - - for m in ( - dataclass_module_1, dataclass_module_1_str, - dataclass_module_2, dataclass_module_2_str, - dataclass_module_3, dataclass_module_3_str, - dataclass_module_4, dataclass_module_4_str, - ): + + for m in (dataclass_module_1, dataclass_module_1_str, + dataclass_module_2, dataclass_module_2_str, + ): with self.subTest(m=m): # There's a difference in how the ClassVars are # interpreted when using string annotations or @@ -4667,14 +4660,6 @@ def custom_dataclass(cls, *args, **kwargs): self.assertEqual(c.x, 10) self.assertEqual(c.__custom__, True) - def test_empty_annotation_string(self): - @dataclass - class DataclassWithEmptyTypeAnnotation: - x: "" - - c = DataclassWithEmptyTypeAnnotation(10) - self.assertEqual(c.x, 10) - class TestReplace(unittest.TestCase): def test(self): diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 574529c70741d8..44fe6f771de2cb 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -576,12 +576,12 @@ def setUp(self): def tearDown(self): try: os.chdir(self.cwd) + try: + shutil.rmtree(self.tempdir) + except: + pass finally: super().tearDown() - try: - shutil.rmtree(self.tempdir) - except: - pass def check_status_and_reason(self, response, status, data=None): def close_conn(): @@ -608,15 +608,6 @@ def close_conn(): reader.close() return body - def check_status_and_headers(self, response, status, headers=None): - # Drain the body so the server-side handler can close the file - # before tearDown removes the tempdir (matters on Windows). - response.read() - self.assertEqual(response.status, status) - if headers: - for name, value in headers.items(): - self.assertEqual(response.getheader(name), value) - def check_list_dir_dirname(self, dirname, quotedname=None): fullpath = os.path.join(self.tempdir, dirname) try: @@ -921,10 +912,9 @@ def test_extra_response_headers_list_dir(self): ('X-Test2', 'test2'), ]): response = self.request(self.base_url + '/') - self.check_status_and_headers(response, HTTPStatus.OK, { - "X-Test1": "test1", - "X-Test2": "test2", - }) + self.assertEqual(response.status, 200) + self.assertEqual(response.getheader("X-Test1"), 'test1') + self.assertEqual(response.getheader("X-Test2"), 'test2') def test_extra_response_headers_get_file(self): with mock.patch.object(self.request_handler, 'extra_response_headers', [ @@ -936,19 +926,18 @@ def test_extra_response_headers_get_file(self): with open(os.path.join(self.tempdir_name, 'index.html'), 'wb') as f: f.write(data) response = self.request(self.base_url + '/') - self.check_status_and_headers(response, HTTPStatus.OK, { - "Set-Cookie": "test1=value1, test2=value2", - "X-Test1": "value3", - }) + self.assertEqual(response.status, 200) + self.assertEqual(response.getheader("Set-Cookie"), + 'test1=value1, test2=value2') + self.assertEqual(response.getheader("X-Test1"), 'value3') def test_extra_response_headers_missing_on_404(self): with mock.patch.object(self.request_handler, 'extra_response_headers', [ ('X-Test1', 'value'), ]): response = self.request(self.base_url + '/missing.html') - self.check_status_and_headers(response, HTTPStatus.NOT_FOUND, { - "X-Test1": None, - }) + self.assertEqual(response.status, 404) + self.assertEqual(response.getheader("X-Test1"), None) def test_extra_response_headers_dont_overwrite_default_headers(self): with mock.patch.object(self.request_handler, 'extra_response_headers', [ @@ -960,21 +949,21 @@ def test_extra_response_headers_dont_overwrite_default_headers(self): # But cookies in the extra_allowed_duplicate_headers are allowed, # including Set-Cookie response = self.request(self.base_url + '/') - self.check_status_and_headers(response, HTTPStatus.OK, { - "Set-Cookie": "test=allowed", - }) + self.assertEqual(response.status, 200) self.assertNotEqual(response.getheader("Content-Type"), 'test/not_allowed') self.assertNotEqual(response.getheader("Server"), 'not_allowed') + self.assertEqual(response.getheader("Set-Cookie"), 'test=allowed') def test_multiple_requests_dont_duplicate_extra_response_headers(self): with mock.patch.object(self.request_handler, 'extra_response_headers', [ ('x-test', 'test-value'), ]): - for _ in range(2): - response = self.request(self.base_url + '/') - self.check_status_and_headers(response, HTTPStatus.OK, { - "x-test": "test-value", - }) + response = self.request(self.base_url + '/') + self.assertEqual(response.status, 200) + self.assertEqual(response.getheader("x-test"), 'test-value') + response = self.request(self.base_url + '/') + self.assertEqual(response.status, 200) + self.assertEqual(response.getheader("x-test"), 'test-value') class SocketlessRequestHandler(SimpleHTTPRequestHandler): diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 4623b7eb4434b0..d4faaaeca00457 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -1,5 +1,3 @@ -import logging -import logging.handlers from pathlib import Path from test.support.import_helper import unload from test.support.warnings_helper import check_warnings @@ -234,6 +232,9 @@ def test_walk_packages_raises_on_string_or_bytes_input(self): list(pkgutil.walk_packages(bytes_input)) def test_name_resolution(self): + import logging + import logging.handlers + success_cases = ( ('os', os), ('os.path', os.path), @@ -321,53 +322,6 @@ def test_name_resolution(self): with self.assertRaises(exc): pkgutil.resolve_name(s) - def test_name_resolution_strict(self): - # PEP 829: strict=True accepts only the pkg.mod:callable form - # (W(.W)*:W(.W)*) -- both the colon and the callable are required. - success_cases = ( - ('os.path:pathsep', os.path.pathsep), - ('logging.handlers:SysLogHandler', - logging.handlers.SysLogHandler), - ('logging.handlers:SysLogHandler.LOG_ALERT', - logging.handlers.SysLogHandler.LOG_ALERT), - ('builtins:int', int), - ('builtins:int.from_bytes', int.from_bytes), - ('os:path', os.path), - ) - - # All of these are accepted under strict=False but must be - # rejected under strict=True. - failure_cases = ( - 'os', # no colon (non-strict form) - 'os.path', # no colon - 'logging:', # colon, empty callable - 'os.foo:', # colon, empty callable - ':int', # empty package - 'os.path:join:extra', # extra colon - 'os.path.9abc:join', # invalid identifier in package - 'os.path:9abc', # invalid identifier in callable - '', # empty - '?abc:foo', # invalid character - ) - - for s, expected in success_cases: - with self.subTest(s=s): - self.assertEqual( - pkgutil.resolve_name(s, strict=True), expected) - - for s in failure_cases: - with self.subTest(s=s): - with self.assertRaises(ValueError): - pkgutil.resolve_name(s, strict=True) - - # Cache independence: a strict=True call must not poison - # strict=False (and vice versa). Exercise both orderings. - self.assertEqual( - pkgutil.resolve_name('os:path', strict=True), os.path) - self.assertEqual(pkgutil.resolve_name('os.path'), os.path) - self.assertEqual( - pkgutil.resolve_name('os:path', strict=True), os.path) - def test_name_resolution_import_rebinding(self): # The same data is also used for testing import in test_import and # mock.patch in test_unittest. diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py index c522c50d1fd5fa..6667953f29abac 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py @@ -15,14 +15,9 @@ "Test only runs when _remote_debugging is available" ) -from test.support import ( - force_not_colorized, - is_emscripten, - requires_remote_subprocess_debugging, -) +from test.support import is_emscripten, requires_remote_subprocess_debugging from profiling.sampling.cli import main -from profiling.sampling.constants import PROFILING_MODE_ALL, PROFILING_MODE_WALL from profiling.sampling.errors import SamplingScriptNotFoundError, SamplingModuleNotFoundError, SamplingUnknownProcessError class TestSampleProfilerCLI(unittest.TestCase): @@ -529,154 +524,6 @@ def test_argument_parsing_basic(self): mock_sample.assert_called_once() - def _run_dump_cli( - self, - *cli_args, - dump_return=None, - dump_side_effect=None, - process_running=True, - ): - """Run main() for a `dump` invocation, returning (mock_dump, mock_print).""" - argv = ["profiling.sampling.cli", "dump", *cli_args] - dump_kwargs = {} - if dump_side_effect is not None: - dump_kwargs["side_effect"] = dump_side_effect - else: - dump_kwargs["return_value"] = [] if dump_return is None else dump_return - with ( - mock.patch("sys.argv", argv), - mock.patch( - "profiling.sampling.cli._is_process_running", - return_value=process_running, - ), - mock.patch("profiling.sampling.cli.dump_stack", **dump_kwargs) as mock_dump_stack, - mock.patch("profiling.sampling.cli.print_stack_dump") as mock_print_stack_dump, - ): - main() - return mock_dump_stack, mock_print_stack_dump - - def _run_dump_cli_expecting_exit(self, *cli_args, capture="stderr"): - """Run main() for a `dump` invocation expected to SystemExit, return (cm, captured_text).""" - argv = ["profiling.sampling.cli", "dump", *cli_args] - buf = io.StringIO() - stream = "sys.stderr" if capture == "stderr" else "sys.stdout" - with ( - mock.patch("sys.argv", argv), - mock.patch(stream, buf), - self.assertRaises(SystemExit) as cm, - ): - main() - return cm, buf.getvalue() - - def test_cli_dump_subcommand(self): - stack_frames = [mock.sentinel.stack_frames] - mock_dump_stack, mock_print_stack_dump = self._run_dump_cli( - "12345", dump_return=stack_frames - ) - - mock_dump_stack.assert_called_once() - self.assertEqual(mock_dump_stack.call_args.args, (12345,)) - call_kwargs = mock_dump_stack.call_args.kwargs - self.assertFalse(call_kwargs["all_threads"]) - self.assertIsNone(call_kwargs["async_aware"]) - self.assertFalse(call_kwargs["native"]) - self.assertTrue(call_kwargs["gc"]) - self.assertFalse(call_kwargs["opcodes"]) - self.assertFalse(call_kwargs["blocking"]) - self.assertEqual(call_kwargs["mode"], PROFILING_MODE_ALL) - mock_print_stack_dump.assert_called_once_with(stack_frames, pid=12345) - - def test_cli_dump_subcommand_options(self): - mock_dump_stack, _ = self._run_dump_cli( - "-a", "--native", "--no-gc", "--opcodes", "--blocking", "12345" - ) - - call_kwargs = mock_dump_stack.call_args.kwargs - self.assertTrue(call_kwargs["all_threads"]) - self.assertTrue(call_kwargs["native"]) - self.assertFalse(call_kwargs["gc"]) - self.assertTrue(call_kwargs["opcodes"]) - self.assertTrue(call_kwargs["blocking"]) - self.assertEqual(call_kwargs["mode"], PROFILING_MODE_ALL) - - def test_cli_dump_rejects_mode_option(self): - cm, stderr = self._run_dump_cli_expecting_exit("12345", "--mode", "cpu") - self.assertEqual(cm.exception.code, 2) - self.assertIn("unrecognized arguments: --mode", stderr) - - def test_cli_dump_async_aware_defaults_to_all(self): - mock_dump_stack, _ = self._run_dump_cli("--async-aware", "12345") - call_kwargs = mock_dump_stack.call_args.kwargs - self.assertEqual(call_kwargs["async_aware"], "all") - self.assertEqual(call_kwargs["mode"], PROFILING_MODE_WALL) - - def test_cli_dump_async_mode_all_is_forwarded(self): - mock_dump_stack, _ = self._run_dump_cli( - "--async-aware", "--async-mode", "all", "12345" - ) - call_kwargs = mock_dump_stack.call_args.kwargs - self.assertEqual(call_kwargs["async_aware"], "all") - self.assertEqual(call_kwargs["mode"], PROFILING_MODE_WALL) - - def test_cli_dump_async_mode_running_is_forwarded(self): - mock_dump_stack, _ = self._run_dump_cli( - "--async-aware", "--async-mode", "running", "12345" - ) - call_kwargs = mock_dump_stack.call_args.kwargs - self.assertEqual(call_kwargs["async_aware"], "running") - self.assertEqual(call_kwargs["mode"], PROFILING_MODE_WALL) - - def test_cli_dump_async_mode_requires_async_aware(self): - cm, stderr = self._run_dump_cli_expecting_exit( - "--async-mode", "running", "12345" - ) - self.assertEqual(cm.exception.code, 2) - self.assertIn("--async-mode requires --async-aware", stderr) - - def test_cli_dump_rejects_async_aware_with_all_threads(self): - cm, stderr = self._run_dump_cli_expecting_exit( - "--async-aware", "-a", "12345" - ) - self.assertEqual(cm.exception.code, 2) - self.assertIn("--all-threads", stderr) - self.assertIn("incompatible with --async-aware", stderr) - - def test_cli_dump_rejects_async_aware_with_native(self): - cm, stderr = self._run_dump_cli_expecting_exit( - "--async-aware", "--native", "12345" - ) - self.assertEqual(cm.exception.code, 2) - self.assertIn("--native", stderr) - self.assertIn("incompatible with --async-aware", stderr) - - def test_cli_dump_rejects_async_aware_with_no_gc(self): - cm, stderr = self._run_dump_cli_expecting_exit( - "--async-aware", "--no-gc", "12345" - ) - self.assertEqual(cm.exception.code, 2) - self.assertIn("--no-gc", stderr) - self.assertIn("incompatible with --async-aware", stderr) - - def test_cli_dump_unknown_process(self): - with self.assertRaises(SamplingUnknownProcessError): - self._run_dump_cli("12345", process_running=False) - - def test_cli_dump_process_exits_before_snapshot(self): - with self.assertRaises(SystemExit) as cm: - self._run_dump_cli("12345", dump_side_effect=ProcessLookupError) - self.assertIn( - "No stack dump collected - process 12345 exited", - str(cm.exception.code), - ) - - @force_not_colorized - def test_cli_dump_help_lists_dump_options_without_mode(self): - cm, stdout = self._run_dump_cli_expecting_exit("--help", capture="stdout") - self.assertEqual(cm.exception.code, 0) - self.assertIn("--async-mode {running,all}", stdout) - self.assertIn("--opcodes", stdout) - self.assertNotIn("--mode {wall,cpu,gil,exception}", stdout) - def test_sort_options(self): sort_options = [ ("nsamples", 0), diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py index 68bc59a5414a05..8d70a1d2ef8cfc 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py @@ -1,16 +1,13 @@ """Tests for sampling profiler core functionality.""" -import contextlib import io import re -import types from unittest import mock import unittest try: import _remote_debugging # noqa: F401 - from profiling.sampling.constants import PROFILING_MODE_ALL, PROFILING_MODE_WALL - from profiling.sampling.sample import SampleProfiler, dump_stack + from profiling.sampling.sample import SampleProfiler from profiling.sampling.pstats_collector import PstatsCollector except ImportError: raise unittest.SkipTest( @@ -66,92 +63,6 @@ def test_sample_profiler_initialization(self): self.assertEqual(profiler.sample_interval_usec, 5000) self.assertEqual(profiler.all_threads, True) - @staticmethod - @contextlib.contextmanager - def _patched_unwinder(): - """Yield a namespace exposing the mock unwinder ``instance`` and ``cls``.""" - instance = mock.MagicMock() - with mock.patch( - "_remote_debugging.RemoteUnwinder", return_value=instance - ) as cls: - yield types.SimpleNamespace(instance=instance, cls=cls) - - def test_dump_stack_uses_single_unwinder_snapshot(self): - stack_frames = [mock.sentinel.stack_frames] - with self._patched_unwinder() as u: - u.instance.get_stack_trace.return_value = stack_frames - result = dump_stack(12345) - - self.assertIs(result, stack_frames) - self.assertEqual(u.cls.call_args.kwargs["mode"], PROFILING_MODE_ALL) - u.instance.get_stack_trace.assert_called_once_with() - - def test_dump_stack_returns_empty_async_snapshot(self): - with self._patched_unwinder() as u: - u.instance.get_async_stack_trace.return_value = [] - result = dump_stack(12345, async_aware="running") - - self.assertEqual(result, []) - u.instance.get_async_stack_trace.assert_called_once_with() - u.instance.get_stack_trace.assert_not_called() - - def test_dump_stack_async_all_uses_all_awaited_by(self): - stack_frames = [mock.sentinel.awaited_info] - with self._patched_unwinder() as u: - u.instance.get_all_awaited_by.return_value = stack_frames - result = dump_stack(12345, async_aware="all") - - self.assertIs(result, stack_frames) - u.instance.get_all_awaited_by.assert_called_once_with() - u.instance.get_stack_trace.assert_not_called() - u.instance.get_async_stack_trace.assert_not_called() - - def test_dump_stack_passes_unwinder_options(self): - with self._patched_unwinder() as u: - u.instance.get_stack_trace.return_value = [] - dump_stack( - 12345, - all_threads=True, - native=True, - gc=False, - opcodes=True, - ) - - call_kwargs = u.cls.call_args.kwargs - self.assertTrue(call_kwargs["all_threads"]) - self.assertEqual(call_kwargs["mode"], PROFILING_MODE_ALL) - self.assertTrue(call_kwargs["native"]) - self.assertFalse(call_kwargs["gc"]) - self.assertTrue(call_kwargs["opcodes"]) - self.assertFalse(call_kwargs["skip_non_matching_threads"]) - self.assertTrue(call_kwargs["cache_frames"]) - - def test_dump_stack_wall_mode_skips_non_matching_threads(self): - with self._patched_unwinder() as u: - u.instance.get_stack_trace.return_value = [] - dump_stack(12345, mode=PROFILING_MODE_WALL) - - self.assertTrue(u.cls.call_args.kwargs["skip_non_matching_threads"]) - - def test_dump_stack_blocking_pauses_and_resumes_threads(self): - stack_frames = [mock.sentinel.stack_frames] - with self._patched_unwinder() as u: - u.instance.get_stack_trace.return_value = stack_frames - result = dump_stack(12345, blocking=True) - - self.assertIs(result, stack_frames) - u.instance.pause_threads.assert_called_once_with() - u.instance.resume_threads.assert_called_once_with() - - def test_dump_stack_blocking_resumes_threads_after_failure(self): - with self._patched_unwinder() as u: - u.instance.get_stack_trace.side_effect = RuntimeError("boom") - with self.assertRaises(RuntimeError): - dump_stack(12345, blocking=True) - - u.instance.pause_threads.assert_called_once_with() - u.instance.resume_threads.assert_called_once_with() - def test_sample_profiler_sample_method_timing(self): """Test that the sample method respects duration and handles timing correctly.""" diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index ac69e2cbdbbe54..e7dc5e2611c2de 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -16,7 +16,6 @@ from test.support.script_helper import spawn_python, kill_python import ast import builtins -import contextlib import glob import io import os @@ -28,7 +27,6 @@ import sysconfig import tempfile from textwrap import dedent -from types import SimpleNamespace import urllib.error import urllib.request from unittest import mock @@ -125,11 +123,14 @@ def test_addpackage(self): # comment or import that is a valid directory name for where the .pth # file resides; invalid directories are not added pth_file = PthFile() - # Ensure we have a clean slate. - pth_file.cleanup(prep=True) - with pth_file.create(): + pth_file.cleanup(prep=True) # to make sure that nothing is + # pre-existing that shouldn't be + try: + pth_file.create() site.addpackage(pth_file.base_dir, pth_file.filename, set()) self.pth_file_tests(pth_file) + finally: + pth_file.cleanup() def make_pth(self, contents, pth_dir='.', pth_name=TESTFN): # Create a .pth file and return its (abspath, basename). @@ -149,6 +150,9 @@ def test_addpackage_import_bad_syntax(self): self.assertRegex(err_out.getvalue(), "line 1") self.assertRegex(err_out.getvalue(), re.escape(os.path.join(pth_dir, pth_fn))) + # XXX: the previous two should be independent checks so that the + # order doesn't matter. The next three could be a single check + # but my regex foo isn't good enough to write it. self.assertRegex(err_out.getvalue(), 'Traceback') self.assertRegex(err_out.getvalue(), r'import bad-syntax') self.assertRegex(err_out.getvalue(), 'SyntaxError') @@ -158,8 +162,10 @@ def test_addpackage_import_bad_exec(self): pth_dir, pth_fn = self.make_pth("randompath\nimport nosuchmodule\n") with captured_stderr() as err_out: site.addpackage(pth_dir, pth_fn, set()) + self.assertRegex(err_out.getvalue(), "line 2") self.assertRegex(err_out.getvalue(), re.escape(os.path.join(pth_dir, pth_fn))) + # XXX: ditto previous XXX comment. self.assertRegex(err_out.getvalue(), 'Traceback') self.assertRegex(err_out.getvalue(), 'ModuleNotFoundError') @@ -172,68 +178,65 @@ def test_addpackage_empty_lines(self): def test_addpackage_import_bad_pth_file(self): # Issue 5258 pth_dir, pth_fn = self.make_pth("abc\x00def\n") + with captured_stderr() as err_out: + self.assertFalse(site.addpackage(pth_dir, pth_fn, set())) + self.maxDiff = None + self.assertEqual(err_out.getvalue(), "") for path in sys.path: if isinstance(path, str): self.assertNotIn("abc\x00def", path) def test_addsitedir(self): - # addsitedir() reads .pth files and, when called standalone - # (known_paths=None), flushes paths and import lines immediately. + # Same tests for test_addpackage since addsitedir() essentially just + # calls addpackage() for every .pth file in the directory pth_file = PthFile() - # Ensure we have a clean slate. - pth_file.cleanup(prep=True) - with pth_file.create(): - site.addsitedir(pth_file.base_dir) - self.pth_file_tests(pth_file) - - def test_addsitedir_explicit_flush(self): - # addsitedir() reads .pth files and, with - # defer_processing_start_files=True, accumulates pending state - # without flushing. A subsequent process_startup_files() call - # then applies the paths and runs the import lines. - pth_file = PthFile() - # Ensure we have a clean slate. - pth_file.cleanup(prep=True) - with pth_file.create(): - # Pass defer_processing_start_files=True to prevent flushing. - site.addsitedir(pth_file.base_dir, set(), - defer_processing_start_files=True) - self.assertNotIn(pth_file.imported, sys.modules) - site.process_startup_files() + pth_file.cleanup(prep=True) # Make sure that nothing is pre-existing + # that is tested for + try: + pth_file.create() + site.addsitedir(pth_file.base_dir, set()) self.pth_file_tests(pth_file) + finally: + pth_file.cleanup() def test_addsitedir_dotfile(self): pth_file = PthFile('.dotfile') - # Ensure we have a clean slate. pth_file.cleanup(prep=True) - with pth_file.create(): - site.addsitedir(pth_file.base_dir) + try: + pth_file.create() + site.addsitedir(pth_file.base_dir, set()) self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() @unittest.skipUnless(hasattr(os, 'chflags'), 'test needs os.chflags()') def test_addsitedir_hidden_flags(self): pth_file = PthFile() - # Ensure we have a clean slate. pth_file.cleanup(prep=True) - with pth_file.create(): + try: + pth_file.create() st = os.stat(pth_file.file_path) os.chflags(pth_file.file_path, st.st_flags | stat.UF_HIDDEN) - site.addsitedir(pth_file.base_dir) + site.addsitedir(pth_file.base_dir, set()) self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() @unittest.skipUnless(sys.platform == 'win32', 'test needs Windows') @support.requires_subprocess() def test_addsitedir_hidden_file_attribute(self): pth_file = PthFile() - # Ensure we have a clean slate. pth_file.cleanup(prep=True) - with pth_file.create(): + try: + pth_file.create() subprocess.check_call(['attrib', '+H', pth_file.file_path]) - site.addsitedir(pth_file.base_dir) + site.addsitedir(pth_file.base_dir, set()) self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() # This tests _getuserbase, hence the double underline # to distinguish from a test for getuserbase @@ -397,7 +400,7 @@ def test_trace(self): self.assertEqual(sys.stderr.getvalue(), out) -class PthFile: +class PthFile(object): """Helper class for handling testing of .pth files""" def __init__(self, filename_base=TESTFN, imported="time", @@ -412,7 +415,6 @@ def __init__(self, filename_base=TESTFN, imported="time", self.good_dir_path = os.path.join(self.base_dir, self.good_dirname) self.bad_dir_path = os.path.join(self.base_dir, self.bad_dirname) - @contextlib.contextmanager def create(self): """Create a .pth file with a comment, blank lines, an ``import ``, a line with self.good_dirname, and a line with @@ -421,7 +423,8 @@ def create(self): Creation of the directory for self.good_dir_path (based off of self.good_dirname) is also performed. - Used as a context manager: self.cleanup() is called on exit. + Make sure to call self.cleanup() to undo anything done by this method. + """ FILE = open(self.file_path, 'w') try: @@ -433,10 +436,6 @@ def create(self): finally: FILE.close() os.mkdir(self.good_dir_path) - try: - yield self - finally: - self.cleanup() def cleanup(self, prep=False): """Make sure that the .pth file is deleted, self.imported is not in @@ -909,544 +908,5 @@ def test_both_args(self): self.assertEqual(output, excepted_output) -class StartFileTests(unittest.TestCase): - """Tests for .start file processing (PEP 829).""" - - def setUp(self): - self.enterContext(import_helper.DirsOnSysPath()) - self.tmpdir = self.sitedir = self.enterContext(os_helper.temp_dir()) - # Save and clear all pending dicts. - self.saved_entrypoints = site._pending_entrypoints.copy() - self.saved_syspaths = site._pending_syspaths.copy() - self.saved_importexecs = site._pending_importexecs.copy() - site._pending_entrypoints.clear() - site._pending_syspaths.clear() - site._pending_importexecs.clear() - - def tearDown(self): - site._pending_entrypoints = self.saved_entrypoints.copy() - site._pending_syspaths = self.saved_syspaths.copy() - site._pending_importexecs = self.saved_importexecs.copy() - - def _make_start(self, content, name='testpkg'): - """Write a .start file and return its basename.""" - basename = f"{name}.start" - filepath = os.path.join(self.tmpdir, basename) - with open(filepath, 'w', encoding='utf-8') as f: - f.write(content) - return basename - - def _make_pth(self, content, name='testpkg'): - """Write a .pth file and return its basename.""" - basename = f"{name}.pth" - filepath = os.path.join(self.tmpdir, basename) - with open(filepath, 'w', encoding='utf-8') as f: - f.write(content) - return basename - - def _all_entrypoints(self): - """Flatten _pending_entrypoints dict into a list of (filename, entry) tuples.""" - result = [] - for filename, entries in site._pending_entrypoints.items(): - for entry in entries: - result.append((filename, entry)) - return result - - def _just_entrypoints(self): - return [entry for filename, entry in self._all_entrypoints()] - - # --- _read_start_file tests --- - - def test_read_start_file_basic(self): - self._make_start("os.path:join\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') - fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], ['os.path:join']) - - def test_read_start_file_multiple_entries(self): - self._make_start("os.path:join\nos.path:exists\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') - fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], - ['os.path:join', 'os.path:exists']) - - def test_read_start_file_comments_and_blanks(self): - self._make_start("# a comment\n\nos.path:join\n \n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') - fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], ['os.path:join']) - - def test_read_start_file_accepts_all_non_blank_lines(self): - # Syntax validation is deferred to entry-point execution time - # (where pkgutil.resolve_name(strict=True) enforces the strict - # pkg.mod:callable form), so parsing accepts every non-blank, - # non-comment line, including syntactically invalid ones. - content = ( - "os.path\n" # no colon - "pkg.mod:\n" # empty callable - ":callable\n" # empty module - "pkg.mod:callable:extra\n" # multiple colons - "os.path:join\n" # valid - ) - self._make_start(content, name='foo') - site._read_start_file(self.sitedir, 'foo.start') - fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], [ - 'os.path', - 'pkg.mod:', - ':callable', - 'pkg.mod:callable:extra', - 'os.path:join', - ]) - - def test_read_start_file_empty(self): - # PEP 829: an empty .start file is still registered as present - # (with an empty entry-point list) so that it suppresses `import` - # lines in any matching .pth file. - self._make_start("", name='foo') - site._read_start_file(self.sitedir, 'foo.start') - fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {fullname: []}) - - def test_read_start_file_comments_only(self): - # As with an empty file, a comments-only .start file is registered - # as present so it can suppress matching .pth `import` lines. - self._make_start("# just a comment\n# another\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') - fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {fullname: []}) - - def test_read_start_file_nonexistent(self): - with captured_stderr(): - site._read_start_file(self.tmpdir, 'nonexistent.start') - self.assertEqual(site._pending_entrypoints, {}) - - @unittest.skipUnless(hasattr(os, 'chflags'), 'test needs os.chflags()') - def test_read_start_file_hidden_flags(self): - self._make_start("os.path:join\n", name='foo') - filepath = os.path.join(self.tmpdir, 'foo.start') - st = os.stat(filepath) - os.chflags(filepath, st.st_flags | stat.UF_HIDDEN) - site._read_start_file(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {}) - - def test_read_start_file_duplicates_not_deduplicated(self): - # PEP 829: duplicate entry points are NOT deduplicated. - self._make_start("os.path:join\nos.path:join\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') - fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], - ['os.path:join', 'os.path:join']) - - def test_read_start_file_accepts_utf8_bom(self): - # PEP 829: .start files MUST be utf-8-sig (UTF-8 with optional BOM). - filepath = os.path.join(self.tmpdir, 'foo.start') - with open(filepath, 'wb') as f: - f.write(b'\xef\xbb\xbf' + b'os.path:join\n') - site._read_start_file(self.sitedir, 'foo.start') - fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual( - site._pending_entrypoints[fullname], ['os.path:join']) - - def test_read_start_file_invalid_utf8_silently_skipped(self): - # PEP 829: .start files MUST be utf-8-sig. Unlike .pth, there is - # no locale-encoding fallback -- a .start file that is not valid - # UTF-8 is silently skipped, with no key registered in - # _pending_entrypoints and no output to stderr (parsing errors - # are reported only under -v). - filepath = os.path.join(self.tmpdir, 'foo.start') - with open(filepath, 'wb') as f: - # Bare continuation byte -- invalid as a UTF-8 start byte. - f.write(b'\x80\x80\x80\n') - with captured_stderr() as err: - site._read_start_file(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {}) - self.assertEqual(err.getvalue(), "") - - def test_two_start_files_with_duplicates_not_deduplicated(self): - self._make_start("os.path:join", name="foo") - self._make_start("os.path:join", name="bar") - site._read_start_file(self.sitedir, 'foo.start') - site._read_start_file(self.sitedir, 'bar.start') - self.assertEqual(self._just_entrypoints(), - ['os.path:join', 'os.path:join']) - - # --- _read_pth_file tests --- - - def test_read_pth_file_paths(self): - subdir = os.path.join(self.sitedir, 'mylib') - os.mkdir(subdir) - self._make_pth("mylib\n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) - fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths[fullname]) - - def test_read_pth_file_imports_collected(self): - self._make_pth("import sys\n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) - fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertEqual(site._pending_importexecs[fullname], ['import sys']) - - def test_read_pth_file_comments_and_blanks(self): - self._make_pth("# comment\n\n \n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) - self.assertEqual(site._pending_syspaths, {}) - self.assertEqual(site._pending_importexecs, {}) - - def test_read_pth_file_deduplication(self): - subdir = os.path.join(self.sitedir, 'mylib') - os.mkdir(subdir) - known_paths = set() - self._make_pth("mylib\n", name='a') - self._make_pth("mylib\n", name='b') - site._read_pth_file(self.sitedir, 'a.pth', known_paths) - site._read_pth_file(self.sitedir, 'b.pth', known_paths) - # Only one entry across both files. - all_dirs = [] - for dirs in site._pending_syspaths.values(): - all_dirs.extend(dirs) - self.assertEqual(all_dirs, [subdir]) - - def test_read_pth_file_bad_line_continues(self): - # PEP 829: errors on individual lines don't abort the file. - subdir = os.path.join(self.sitedir, 'goodpath') - os.mkdir(subdir) - self._make_pth("abc\x00def\ngoodpath\n", name='foo') - with captured_stderr(): - site._read_pth_file(self.sitedir, 'foo.pth', set()) - fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) - - def _flags_with_verbose(self, verbose): - # Build a sys.flags clone with verbose overridden but every - # other field preserved, so unrelated reads like - # sys.flags.optimize during io.open_code() continue to work. - attrs = {name: getattr(sys.flags, name) - for name in sys.flags.__match_args__} - attrs['verbose'] = verbose - return SimpleNamespace(**attrs) - - def test_read_pth_file_parse_error_silent_by_default(self): - # PEP 829: parse-time errors are silent unless -v is given. - # Force the error path by making makepath() raise. - self._make_pth("badline\n", name='foo') - with mock.patch('site.makepath', side_effect=ValueError("boom")), \ - mock.patch('sys.flags', self._flags_with_verbose(False)), \ - captured_stderr() as err: - site._read_pth_file(self.sitedir, 'foo.pth', set()) - self.assertEqual(err.getvalue(), "") - - def test_read_pth_file_parse_error_reported_under_verbose(self): - # PEP 829: parse-time errors are reported when -v is given. - self._make_pth("badline\n", name='foo') - with mock.patch('site.makepath', side_effect=ValueError("boom")), \ - mock.patch('sys.flags', self._flags_with_verbose(True)), \ - captured_stderr() as err: - site._read_pth_file(self.sitedir, 'foo.pth', set()) - out = err.getvalue() - self.assertIn('Error in', out) - self.assertIn('foo.pth', out) - - def test_read_pth_file_locale_fallback(self): - # PEP 829: .pth files that fail UTF-8 decoding fall back to the - # locale encoding for backward compatibility (deprecated in - # 3.15, to be removed in 3.20). Mock locale.getencoding() so - # the test does not depend on the host's actual locale. - subdir = os.path.join(self.sitedir, 'mylib') - os.mkdir(subdir) - filepath = os.path.join(self.tmpdir, 'foo.pth') - # \xe9 is invalid UTF-8 but valid in latin-1. - with open(filepath, 'wb') as f: - f.write(b'# caf\xe9 comment\nmylib\n') - with mock.patch('locale.getencoding', return_value='latin-1'), \ - captured_stderr(): - site._read_pth_file(self.sitedir, 'foo.pth', set()) - fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) - - # --- _execute_start_entrypoints tests --- - - def test_execute_entrypoints_with_callable(self): - # Entrypoint with callable is invoked. - mod_dir = os.path.join(self.sitedir, 'epmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ -called = False -def startup(): - global called - called = True -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'epmod', None) - fullname = os.path.join(self.sitedir, 'epmod.start') - site._pending_entrypoints[fullname] = ['epmod:startup'] - site._execute_start_entrypoints() - import epmod - self.assertTrue(epmod.called) - - def test_execute_entrypoints_import_error(self): - # Import error prints traceback but continues. - fullname = os.path.join(self.sitedir, 'bad.start') - site._pending_entrypoints[fullname] = [ - 'nosuchmodule_xyz:func', 'os.path:join'] - with captured_stderr() as err: - site._execute_start_entrypoints() - self.assertIn('nosuchmodule_xyz', err.getvalue()) - # os.path:join should still have been called (no exception for it) - - def test_execute_entrypoints_strict_syntax_rejection(self): - # PEP 829: only the strict pkg.mod:callable form is valid. - # At entry-point execution, pkgutil.resolve_name(strict=True) - # raises ValueError for invalid syntax; the invalid entry is - # reported and execution continues with the next one. - fullname = os.path.join(self.sitedir, 'bad.start') - site._pending_entrypoints[fullname] = [ - 'os.path', # no colon - 'pkg.mod:', # empty callable - ':callable', # empty module - 'pkg.mod:callable:extra', # multiple colons - ] - with captured_stderr() as err: - site._execute_start_entrypoints() - out = err.getvalue() - self.assertIn('Invalid entry point syntax', out) - for bad in ('os.path', 'pkg.mod:', ':callable', - 'pkg.mod:callable:extra'): - self.assertIn(bad, out) - - def test_execute_entrypoints_callable_error(self): - # Callable that raises prints traceback but continues. - mod_dir = os.path.join(self.sitedir, 'badmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ -def fail(): - raise RuntimeError("boom") -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'badmod', None) - fullname = os.path.join(self.sitedir, 'badmod.start') - site._pending_entrypoints[fullname] = ['badmod:fail'] - with captured_stderr() as err: - site._execute_start_entrypoints() - self.assertIn('RuntimeError', err.getvalue()) - self.assertIn('boom', err.getvalue()) - - def test_execute_entrypoints_duplicates_called_twice(self): - # PEP 829: duplicate entry points execute multiple times. - mod_dir = os.path.join(self.sitedir, 'countmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ -call_count = 0 -def bump(): - global call_count - call_count += 1 -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'countmod', None) - fullname = os.path.join(self.sitedir, 'countmod.start') - site._pending_entrypoints[fullname] = [ - 'countmod:bump', 'countmod:bump'] - site._execute_start_entrypoints() - import countmod - self.assertEqual(countmod.call_count, 2) - - # --- _exec_imports tests --- - - def test_exec_imports_suppressed_by_matching_start(self): - # Import lines from foo.pth are suppressed when foo.start exists. - pth_fullname = os.path.join(self.sitedir, 'foo.pth') - start_fullname = os.path.join(self.sitedir, 'foo.start') - site._pending_importexecs[pth_fullname] = ['import sys'] - site._pending_entrypoints[start_fullname] = ['os.path:join'] - # Should not exec the import line; no error expected. - site._exec_imports() - - def test_exec_imports_not_suppressed_by_different_start(self): - # Import lines from foo.pth are NOT suppressed by bar.start. - pth_fullname = os.path.join(self.sitedir, 'foo.pth') - start_fullname = os.path.join(self.sitedir, 'bar.start') - site._pending_importexecs[pth_fullname] = ['import sys'] - site._pending_entrypoints[start_fullname] = ['os.path:join'] - # Should execute the import line without error. - site._exec_imports() - - def test_exec_imports_suppressed_by_empty_matching_start(self): - self._make_start("", name='foo') - self._make_pth("import epmod; epmod.startup()", name='foo') - mod_dir = os.path.join(self.sitedir, 'epmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ -called = False -def startup(): - global called - called = True -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'epmod', None) - site._read_pth_file(self.sitedir, 'foo.pth', set()) - site._read_start_file(self.sitedir, 'foo.start') - site._exec_imports() - import epmod - self.assertFalse(epmod.called) - - # --- _extend_syspath tests --- - - def test_extend_syspath_existing_dir(self): - subdir = os.path.join(self.sitedir, 'extlib') - os.mkdir(subdir) - site._pending_syspaths['test.pth'] = [subdir] - site._extend_syspath() - self.assertIn(subdir, sys.path) - - def test_extend_syspath_nonexistent_dir(self): - nosuch = os.path.join(self.sitedir, 'nosuchdir') - site._pending_syspaths['test.pth'] = [nosuch] - with captured_stderr() as err: - site._extend_syspath() - self.assertNotIn(nosuch, sys.path) - self.assertIn('does not exist', err.getvalue()) - - # --- addsitedir integration tests --- - - def test_addsitedir_discovers_start_files(self): - # addsitedir() should discover .start files and accumulate entries. - self._make_start("os.path:join\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - fullname = os.path.join(self.sitedir, 'foo.start') - self.assertIn('os.path:join', site._pending_entrypoints[fullname]) - - def test_addsitedir_start_suppresses_pth_imports(self): - # When foo.start exists, import lines in foo.pth are skipped - # at flush time by _exec_imports(). - self._make_start("os.path:join\n", name='foo') - self._make_pth("import sys\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - pth_fullname = os.path.join(self.sitedir, 'foo.pth') - start_fullname = os.path.join(self.sitedir, 'foo.start') - # Import line was collected... - self.assertIn('import sys', - site._pending_importexecs.get(pth_fullname, [])) - # ...but _exec_imports() will skip it because foo.start exists. - site._exec_imports() - - def test_addsitedir_pth_paths_still_work_with_start(self): - # Path lines in .pth files still work even when a .start file exists. - subdir = os.path.join(self.sitedir, 'mylib') - os.mkdir(subdir) - self._make_start("os.path:join\n", name='foo') - self._make_pth("mylib\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) - - def test_addsitedir_start_alphabetical_order(self): - # Multiple .start files are discovered alphabetically. - self._make_start("os.path:join\n", name='zzz') - self._make_start("os.path:exists\n", name='aaa') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - all_entries = self._all_entrypoints() - entries = [entry for _, entry in all_entries] - idx_a = entries.index('os.path:exists') - idx_z = entries.index('os.path:join') - self.assertLess(idx_a, idx_z) - - def test_addsitedir_pth_before_start(self): - # PEP 829: .pth files are scanned before .start files. - # Create a .pth and .start with the same basename; verify - # the .pth data is collected before .start data. - subdir = os.path.join(self.sitedir, 'mylib') - os.mkdir(subdir) - self._make_pth("mylib\n", name='foo') - self._make_start("os.path:join\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - # Both should be collected. - pth_fullname = os.path.join(self.sitedir, 'foo.pth') - start_fullname = os.path.join(self.sitedir, 'foo.start') - self.assertIn(subdir, site._pending_syspaths.get(pth_fullname, [])) - self.assertIn('os.path:join', - site._pending_entrypoints.get(start_fullname, [])) - - def test_addsitedir_dotfile_start_ignored(self): - # .start files starting with '.' are skipped. Defer flushing so - # the assertion against _pending_entrypoints is meaningful; - # otherwise process_startup_files() would clear the dict - # regardless of whether the dotfile was picked up. - self._make_start("os.path:join\n", name='.hidden') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - self.assertEqual(site._pending_entrypoints, {}) - - def test_addsitedir_standalone_flushes(self): - # When called with known_paths=None (standalone), addsitedir - # flushes immediately so the caller sees the effect. - subdir = os.path.join(self.sitedir, 'flushlib') - os.mkdir(subdir) - self._make_pth("flushlib\n", name='foo') - site.addsitedir(self.sitedir) # known_paths=None - self.assertIn(subdir, sys.path) - # Pending dicts should be cleared after flush. - self.assertEqual(site._pending_syspaths, {}) - - def test_addsitedir_defer_does_not_flush(self): - # With defer_processing_start_files=True, addsitedir accumulates - # pending state but does not flush; sys.path is updated only when - # process_startup_files() is called explicitly. - subdir = os.path.join(self.sitedir, 'acclib') - os.mkdir(subdir) - self._make_pth("acclib\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - # Path is pending, not yet on sys.path. - self.assertNotIn(subdir, sys.path) - fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) - - def test_pth_path_is_available_to_start_entrypoint(self): - # Core PEP 829 invariant: all .pth path extensions are applied to - # sys.path *before* any .start entry point runs, so an entry - # point may live in a module reachable only via a .pth-extended - # path. If the flush phases were inverted, resolving the entry - # point would fail with ModuleNotFoundError. - extdir = os.path.join(self.sitedir, 'extdir') - os.mkdir(extdir) - modpath = os.path.join(extdir, 'mod.py') - with open(modpath, 'w') as f: - f.write("""\ -called = False -def hook(): - global called - called = True -""") - self.addCleanup(sys.modules.pop, 'mod', None) - - # extdir is not on sys.path; only the .pth file makes it so. - self.assertNotIn(extdir, sys.path) - self._make_pth("extdir\n", name='extlib') - self._make_start("mod:hook\n", name='extlib') - - # Standalone addsitedir() triggers the full flush sequence. - site.addsitedir(self.sitedir) - - self.assertIn(extdir, sys.path) - import mod - self.assertTrue( - mod.called, - "entry point did not run; .pth path was likely not applied " - "before .start entry-point execution") - - if __name__ == "__main__": unittest.main() diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 593fa01bf25ed1..c96367f57fb3f2 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -70,21 +70,21 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "cf01946f3a61ba45a08c1e35b223d41d23963e3df5ac98cbad6c8fa5a81070ca" + "checksumValue": "619b30acf7d9b13c9d0ba90d17349e8b524c380cd23d39334b143f74dc4e5ec9" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.5.6.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.5.5.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:openssl:openssl:3.5.6:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.5.5:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "openssl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.5.6" + "versionInfo": "3.5.5" }, { "SPDXID": "SPDXRef-PACKAGE-sqlite", diff --git a/Modules/_ssl_data_36.h b/Modules/_ssl_data_36.h index e1c1eb30ff6a7b..5a2e0d067e2dc7 100644 --- a/Modules/_ssl_data_36.h +++ b/Modules/_ssl_data_36.h @@ -1,6 +1,6 @@ /* File generated by Tools/ssl/make_ssl_data.py */ -/* Generated on 2026-05-03T19:50:43.034653+00:00 */ -/* Generated from Git commit openssl-3.6.2-0-gfe686e15d */ +/* Generated on 2026-02-13T18:19:19.227109+00:00 */ +/* Generated from Git commit openssl-3.6.1-0-gc9a9e5b10 */ /* generated from args.lib2errnum */ static struct py_ssl_library_code library_codes[] = { @@ -4263,11 +4263,6 @@ static struct py_ssl_error_code error_codes[] = { #else {"CONNECT_FAILURE", 61, 100}, #endif - #ifdef HTTP_R_CONTENT_TYPE_MISMATCH - {"CONTENT_TYPE_MISMATCH", ERR_LIB_HTTP, HTTP_R_CONTENT_TYPE_MISMATCH}, - #else - {"CONTENT_TYPE_MISMATCH", 61, 131}, - #endif #ifdef HTTP_R_ERROR_PARSING_ASN1_LENGTH {"ERROR_PARSING_ASN1_LENGTH", ERR_LIB_HTTP, HTTP_R_ERROR_PARSING_ASN1_LENGTH}, #else diff --git a/Objects/abstract.c b/Objects/abstract.c index 359c6ec26559a6..bc1bf8d21891b4 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -247,7 +247,7 @@ PyObject_SetItem(PyObject *o, PyObject *key, PyObject *value) return res; } - { + if (Py_TYPE(o)->tp_as_sequence != &_Py_empty_sequence_methods) { if (_PyIndex_Check(key)) { Py_ssize_t key_value; key_value = PyNumber_AsSsize_t(key, PyExc_IndexError); @@ -281,7 +281,7 @@ PyObject_DelItem(PyObject *o, PyObject *key) return res; } - { + if (Py_TYPE(o)->tp_as_sequence != &_Py_empty_sequence_methods) { if (_PyIndex_Check(key)) { Py_ssize_t key_value; key_value = PyNumber_AsSsize_t(key, PyExc_IndexError); @@ -1316,16 +1316,20 @@ PyNumber_InPlaceMultiply(PyObject *v, PyObject *w) PySequenceMethods *mv = Py_TYPE(v)->tp_as_sequence; PySequenceMethods *mw = Py_TYPE(w)->tp_as_sequence; Py_DECREF(result); - f = mv->sq_inplace_repeat; - if (f == NULL) - f = mv->sq_repeat; - if (f != NULL) - return sequence_repeat(f, v, w); - /* Note that the right hand operand should not be - * mutated in this case so sq_inplace_repeat is not - * used. */ - if (mw->sq_repeat) - return sequence_repeat(mw->sq_repeat, w, v); + if (mv != (PySequenceMethods *)&_Py_empty_sequence_methods) { + f = mv->sq_inplace_repeat; + if (f == NULL) + f = mv->sq_repeat; + if (f != NULL) + return sequence_repeat(f, v, w); + } + else if (mw != (PySequenceMethods *)&_Py_empty_sequence_methods) { + /* Note that the right hand operand should not be + * mutated in this case so sq_inplace_repeat is not + * used. */ + if (mw->sq_repeat) + return sequence_repeat(mw->sq_repeat, w, v); + } result = binop_type_error(v, w, "*="); } return result; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 3c69a60a930503..fe70e2a6950b93 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8720,10 +8720,11 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) /* This won't inherit indirect slots (from tp_as_number etc.) if type doesn't provide the space. */ - if (type->tp_as_number != NULL && base->tp_as_number != NULL) { + if (type->tp_as_number != NULL && base->tp_as_number != NULL + && base->tp_as_number != &_Py_empty_number_methods) { basebase = base->tp_base; - // basebase is NULL when base is 'object' (tp_base == NULL). - if (basebase == NULL || basebase->tp_as_number == NULL) + if (basebase->tp_as_number == NULL + || basebase->tp_as_number == &_Py_empty_number_methods) basebase = NULL; COPYNUM(nb_add); COPYNUM(nb_subtract); @@ -8771,10 +8772,11 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) COPYASYNC(am_anext); } - if (type->tp_as_sequence != NULL && base->tp_as_sequence != NULL) { + if (type->tp_as_sequence != NULL && base->tp_as_sequence != NULL + && base->tp_as_sequence != &_Py_empty_sequence_methods) { basebase = base->tp_base; - // basebase is NULL when base is 'object' (tp_base == NULL). - if (basebase == NULL || basebase->tp_as_sequence == NULL) + if (basebase->tp_as_sequence == NULL + || basebase->tp_as_sequence == &_Py_empty_sequence_methods) basebase = NULL; COPYSEQ(sq_length); COPYSEQ(sq_concat); @@ -8786,10 +8788,11 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) COPYSEQ(sq_inplace_repeat); } - if (type->tp_as_mapping != NULL && base->tp_as_mapping != NULL) { + if (type->tp_as_mapping != NULL && base->tp_as_mapping != NULL + && base->tp_as_mapping != &_Py_empty_mapping_methods) { basebase = base->tp_base; - // basebase is NULL when base is 'object' (tp_base == NULL). - if (basebase == NULL || basebase->tp_as_mapping == NULL) + if (basebase->tp_as_mapping == NULL + || basebase->tp_as_mapping == &_Py_empty_mapping_methods) basebase = NULL; COPYMAP(mp_length); COPYMAP(mp_subscript); @@ -9162,9 +9165,9 @@ type_ready_mro(PyTypeObject *type, int initial) } -static const PyNumberMethods _Py_empty_number_methods = {0}; -static const PySequenceMethods _Py_empty_sequence_methods = {0}; -static const PyMappingMethods _Py_empty_mapping_methods = {0}; +const PyNumberMethods _Py_empty_number_methods = {0}; +const PySequenceMethods _Py_empty_sequence_methods = {0}; +const PyMappingMethods _Py_empty_mapping_methods = {0}; // For static types, inherit tp_as_xxx structures from the base class // if it's NULL. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 405285b65dd270..f80a025fb3bc78 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.5.6 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.5.5 set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.50.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 @@ -79,7 +79,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.5.6 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.5.5 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 diff --git a/PCbuild/python.props b/PCbuild/python.props index f29f3d18de5f9d..3ad8d81dfc9a95 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -82,8 +82,8 @@ $(libffiDir)$(ArchName)\ $(libffiOutDir)include $(ExternalsDir)\mpdecimal-4.0.0\ - $(ExternalsDir)openssl-3.5.6\ - $(ExternalsDir)openssl-bin-3.5.6\$(ArchName)\ + $(ExternalsDir)openssl-3.5.5\ + $(ExternalsDir)openssl-bin-3.5.5\$(ArchName)\ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ $(ExternalsDir)\zlib-1.3.1\ diff --git a/Python/specialize.c b/Python/specialize.c index 793bac58adf41a..32c49213a76354 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2818,15 +2818,15 @@ static int check_type_always_true(PyTypeObject *ty) { PyNumberMethods *nb = ty->tp_as_number; - if (nb && nb->nb_bool) { + if (nb->nb_bool) { return SPEC_FAIL_TO_BOOL_NUMBER; } PyMappingMethods *mp = ty->tp_as_mapping; - if (mp && mp->mp_length) { + if (mp->mp_length) { return SPEC_FAIL_TO_BOOL_MAPPING; } PySequenceMethods *sq = ty->tp_as_sequence; - if (sq && sq->sq_length) { + if (sq->sq_length) { return SPEC_FAIL_TO_BOOL_SEQUENCE; } return 0; diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 1b2cb84f9a81c4..aa89e312b62482 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -352,9 +352,6 @@ Objects/typeobject.c - name_op - Objects/typeobject.c - slotdefs - # It initialized only once when main interpeter starts Objects/typeobject.c - slotdefs_dups - -Objects/typeobject.c - _Py_empty_mapping_methods - -Objects/typeobject.c - _Py_empty_number_methods - -Objects/typeobject.c - _Py_empty_sequence_methods - Objects/unicodeobject.c - stripfuncnames - Objects/unicodeobject.c - utf7_category - Objects/unicodeobject.c unicode_decode_call_errorhandler_wchar argparse - diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 6be1a5ae94ebc6..48207e5330fa90 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -49,11 +49,11 @@ ] OPENSSL_RECENT_VERSIONS = [ - "3.0.20", - "3.3.7", - "3.4.5", - "3.5.6", - "3.6.2", + "3.0.19", + "3.3.6", + "3.4.4", + "3.5.5", + "3.6.1", # See make_ssl_data.py for notes on adding a new version. ]